import numpy as np
import tensorflow as tf
from keras import regularizers
from tensorflow import keras
from tensorflow.keras import layers
from pathlib import Path
import random
import cv2
import matplotlib.pyplot as plt
from src.notebooks.nutils import *
from sklearn.model_selection import train_test_split
from collections import Counter
import json
import os
import time
from pprint import pprint as pp
import itertools
As it was our first approach with image classification, we took up the suggestion and decided to use Caltech 101 dataset.
To download the dataset, one can use commands written below.
# !pip install -r requirements.txt
!dvc update src/data/Caltech-101.zip.dvc
# !python src/scripts/unzip.py src/data/Caltech-101.zip
'src/data/Caltech-101.zip.dvc' didn't change, skipping core>
We haved removed class BACKGROUND_GOOGlE manually, upon inspection, the category appears to consist of random images downloaded from google without any patterns or distinct futures, also the class contains a lot of images simillar to other classes.
!rm -r src/data/Caltech-101/BACKGROUND_Google
rm: cannot remove 'src/data/Caltech-101/BACKGROUND_Google': No such file or directory
Loading the data:
#Selects classes of images with at least down_sample number of images per classes. If the class have more than
#down_sample of images, it allows to select for up to delta number of images. Images are selected randomly.
#If down_sample is None and num_of_classes is not None than it selects a given number of classes and sets
#down_sample to size of smallest class.
#If both are specified, the function selects up to num_of_classes classes and downsamples them to down_sample.
random.seed(12)
def select_images(path, down_sample, delta=10, num_of_classes= None):
labels = [folder.stem for folder in path.iterdir()]
files = [list((path / label).iterdir()) for label in labels]
if not num_of_classes and down_sample:
selector = [i for i in range(len(labels)) if len(files[i]) >= down_sample]
return [labels[i] for i in selector], [random.sample(files[i], min(down_sample + delta, len(files[i]))) for i in
selector]
if num_of_classes and not down_sample:
selector = sorted([i for i in range(len(labels))], reverse = True, key = lambda x: len(files[x]))[:num_of_classes]
down_sample = min([len(files[i]) for i in selector])
return [labels[i] for i in selector], [random.sample(files[i], min(down_sample + delta, len(files[i]))) for i in
selector]
if num_of_classes and down_sample:
selector = sorted([i for i in range(len(labels)) if len(files[i]) >= down_sample], reverse = True, key = lambda x: len(files[x]))[:num_of_classes]
return [labels[i] for i in selector], [random.sample(files[i], min(down_sample + delta, len(files[i]))) for i in
selector]
raise ValueError("Either down_sample or num_of_classes must be not None")
#Load images and resizes them to chosen resolution.
def load_images(labels, files, resolution=(64, 64)):
paths = [file for file_group in files for file in file_group]
loaded_files = np.array([cv2.resize(cv2.imread(str(path)), resolution) for path in paths])
labels = [labels[i] for i in range(0, len(labels)) for _ in range(0, len(files[i]))]
return labels, loaded_files
#Selection of classes, images and creation of maps.
labels, files = select_images(Path("./src/data/Caltech-101"), down_sample=80)
LABEL_ID_MAP = {label: i for i, label in enumerate(labels)}
ID_LABEL_MAP = {i: label for i, label in enumerate(labels)}
CLASS_SIZE = {label: len(files[i]) for i, label in enumerate(labels)}
labels, not_normalized_images = load_images(labels, files)
show_image(labels[10], not_normalized_images[10])
def standardize_images(files):
means = np.mean(files, axis=(0, 1, 2))
std = np.std(files, axis=(0, 1, 2))
return (files - means) / std
standardized_images = standardize_images(not_normalized_images)
show_image(labels[10], standardized_images[10])
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
y = np.array([LABEL_ID_MAP[label] for label in labels])
X_train, X_test, y_train, y_test = train_test_split(standardized_images, y, stratify=y, test_size=0.3, random_state=12)
print("Shapes of X_train and X_test", X_train.shape, X_test.shape)
print("Chapes of y_train and y_test", y_train.shape, y_test.shape)
#Checking the proportions of classes in particular subsets (train and test):
count_train = np.array([x[1] for x in sorted(Counter(y_train).items(), key=lambda el:el[0])])
count_test = np.array([x[1] for x in sorted(Counter(y_test).items(), key=lambda el:el[0])])
print(count_train)
print(count_test)
print(count_train/(count_train+count_test))
Shapes of X_train and X_test (1592, 64, 64, 3) (683, 64, 64, 3) Chapes of y_train and y_test (1592,) (683,) [62 59 63 63 63 59 63 63 63 63 61 57 63 63 60 59 56 63 60 60 63 57 60 63 63 63] [26 26 27 27 27 25 27 27 27 27 26 25 27 27 26 26 24 27 26 26 27 24 25 27 27 27] [0.70454545 0.69411765 0.7 0.7 0.7 0.70238095 0.7 0.7 0.7 0.7 0.70114943 0.69512195 0.7 0.7 0.69767442 0.69411765 0.7 0.7 0.69767442 0.69767442 0.7 0.7037037 0.70588235 0.7 0.7 0.7 ]
Proportions are correct.
show_image(ID_LABEL_MAP[y_train[10]], X_train[10])
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
# Defining our own convolution block.
class CNNBlock(keras.layers.Layer):
def __init__(self, number_of_filters=32, filter_size=(3, 3), activation_function="relu",
dropout=0.2, pool_size=(2, 2)):
super(CNNBlock, self).__init__()
self.number_of_filters = number_of_filters
self.filter_size = filter_size
self.activation_function = activation_function
self.dropout = dropout
self.pool_size = pool_size
self.block = keras.Sequential([
layers.Conv2D(number_of_filters, filter_size, padding='same', activation=self.activation_function),
layers.BatchNormalization(),
layers.Dropout(self.dropout),
layers.MaxPool2D(pool_size=self.pool_size)])
def call(self, inputs):
return self.block(inputs)
def get_config(self):
return {"number_of_filters": self.number_of_filters, "filter_size": self.filter_size,
"activation_function": self.activation_function, "dropout": self.dropout, "pool_size": self.pool_size}
@classmethod
def from_config(cls, config):
return cls(**config)
# Function that returns the final model.
def create_model(len_of_ID_LABEL_MAP, filter_numbers, filter_sizes,
activations, dropout, pool_sizes,
dense_sizes, dense_activations):
assert len(filter_numbers) == len(filter_sizes) == len(activations) == len(dropout) == len(pool_sizes)
assert len(dense_activations) == len(dense_sizes)
convolution_layers = [CNNBlock(number_of_filters=filter_numbers[x],
filter_size=filter_sizes[x], activation_function=activations[x],
dropout=dropout[x], pool_size=pool_sizes[x]) for x in range(0, len(filter_numbers))]
classic_dense_layers = [keras.layers.Dense(dense_sizes[i], dense_activations[i]) for i in range(len(dense_sizes))]
return keras.Sequential(
convolution_layers + [keras.layers.Flatten()] + classic_dense_layers + [keras.layers.Dense(len_of_ID_LABEL_MAP),
keras.layers.Softmax()])
#CNN block params:
filter_numbers = [32, 32, 32]
filter_sizes = [(3, 3), (3, 3), (3,3)]
activations = ["relu", "relu", "relu"]
dropout = [0.2, 0.2, 0.2]
pool_sizes = [(2, 2), (2, 2), (2,2)]
#Dense layers params:
dense_sizes = [256, 256]
dense_activations = ["sigmoid", "relu"]
model = create_model(len(ID_LABEL_MAP), filter_numbers, filter_sizes, activations, dropout, pool_sizes, dense_sizes, dense_activations)
model.compile(optimizer="adam",
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=["accuracy"])
model.evaluate(X_train, y_train)
50/50 [==============================] - 0s 3ms/step - loss: 3.2583 - accuracy: 0.0396
[3.2582859992980957, 0.03957286477088928]
Accuracy is usually between 3-5%
def plot_accuracy(history=None):
plt.plot(history.history['accuracy'],label="accuracy",)
plt.plot(history.history['val_accuracy'],label = "val_accuracy")
plt.yticks(np.linspace(0, 1, num=11))
plt.axhline(y=np.max(history.history['val_accuracy']), color='g', linestyle='--', label="Best accuracy")
plt.legend()
plt.title("Accuracy")
plt.show()
def plot_loss(history=None):
plt.plot(history.history['loss'],label="loss",)
plt.plot(history.history['val_loss'],label = "loss")
plt.legend()
plt.title("Loss")
plt.show()
def plot_confussion_matrix(model=None, X_test=None):
if model is not None:
logits = model.predict(X_test)
y_pred = np.argmax(model.predict(X_test), axis=1)
confussion_matrix = tf.math.confusion_matrix(y_test,y_pred)
plt.imshow(confussion_matrix)
def plot_confussion_matrix2(model, X_test, y_test, classes,
title='Confusion matrix',
cmap=plt.cm.Blues):
logits = model.predict(X_test)
y_pred = np.argmax(model.predict(X_test), axis=1)
cm = tf.math.confusion_matrix(y_test,y_pred)
plt.figure(figsize=(14, 14))
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
history = model.fit(X_train, y_train,
epochs=250,
validation_data=(X_test, y_test),
batch_size=32,
callbacks=[
keras.callbacks.EarlyStopping(
monitor="val_accuracy",
patience=6 # 6
)
])
Epoch 1/250 50/50 [==============================] - 1s 14ms/step - loss: 3.1727 - accuracy: 0.1520 - val_loss: 3.1795 - val_accuracy: 0.1816 Epoch 2/250 50/50 [==============================] - 1s 13ms/step - loss: 2.9962 - accuracy: 0.3392 - val_loss: 3.1913 - val_accuracy: 0.1230 Epoch 3/250 50/50 [==============================] - 1s 13ms/step - loss: 2.8631 - accuracy: 0.4799 - val_loss: 3.1048 - val_accuracy: 0.2255 Epoch 4/250 50/50 [==============================] - 1s 13ms/step - loss: 2.7808 - accuracy: 0.5603 - val_loss: 3.0237 - val_accuracy: 0.3338 Epoch 5/250 50/50 [==============================] - 1s 13ms/step - loss: 2.7212 - accuracy: 0.6181 - val_loss: 2.9446 - val_accuracy: 0.3953 Epoch 6/250 50/50 [==============================] - 1s 13ms/step - loss: 2.6645 - accuracy: 0.6759 - val_loss: 2.8691 - val_accuracy: 0.4671 Epoch 7/250 50/50 [==============================] - 1s 13ms/step - loss: 2.6199 - accuracy: 0.7192 - val_loss: 2.8094 - val_accuracy: 0.5447 Epoch 8/250 50/50 [==============================] - 1s 13ms/step - loss: 2.5783 - accuracy: 0.7613 - val_loss: 2.7566 - val_accuracy: 0.5930 Epoch 9/250 50/50 [==============================] - 1s 13ms/step - loss: 2.5450 - accuracy: 0.7933 - val_loss: 2.6963 - val_accuracy: 0.6471 Epoch 10/250 50/50 [==============================] - 1s 13ms/step - loss: 2.5294 - accuracy: 0.8021 - val_loss: 2.7056 - val_accuracy: 0.6310 Epoch 11/250 50/50 [==============================] - 1s 13ms/step - loss: 2.5150 - accuracy: 0.8122 - val_loss: 2.6976 - val_accuracy: 0.6442 Epoch 12/250 50/50 [==============================] - 1s 13ms/step - loss: 2.5089 - accuracy: 0.8216 - val_loss: 2.7158 - val_accuracy: 0.6310 Epoch 13/250 50/50 [==============================] - 1s 13ms/step - loss: 2.5002 - accuracy: 0.8291 - val_loss: 2.6717 - val_accuracy: 0.6691 Epoch 14/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4818 - accuracy: 0.8499 - val_loss: 2.6623 - val_accuracy: 0.6779 Epoch 15/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4705 - accuracy: 0.8574 - val_loss: 2.6768 - val_accuracy: 0.6618 Epoch 16/250 50/50 [==============================] - 1s 14ms/step - loss: 2.4566 - accuracy: 0.8719 - val_loss: 2.6546 - val_accuracy: 0.6896 Epoch 17/250 50/50 [==============================] - 1s 14ms/step - loss: 2.4400 - accuracy: 0.8901 - val_loss: 2.6376 - val_accuracy: 0.7042 Epoch 18/250 50/50 [==============================] - 1s 14ms/step - loss: 2.4326 - accuracy: 0.8970 - val_loss: 2.6567 - val_accuracy: 0.6823 Epoch 19/250 50/50 [==============================] - 1s 14ms/step - loss: 2.4248 - accuracy: 0.9020 - val_loss: 2.6189 - val_accuracy: 0.7130 Epoch 20/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4172 - accuracy: 0.9077 - val_loss: 2.6323 - val_accuracy: 0.7086 Epoch 21/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4163 - accuracy: 0.9083 - val_loss: 2.6320 - val_accuracy: 0.7028 Epoch 22/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4130 - accuracy: 0.9102 - val_loss: 2.6111 - val_accuracy: 0.7262 Epoch 23/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4094 - accuracy: 0.9133 - val_loss: 2.6180 - val_accuracy: 0.7145 Epoch 24/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4085 - accuracy: 0.9133 - val_loss: 2.6137 - val_accuracy: 0.7218 Epoch 25/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4082 - accuracy: 0.9139 - val_loss: 2.6146 - val_accuracy: 0.7174 Epoch 26/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4073 - accuracy: 0.9152 - val_loss: 2.6369 - val_accuracy: 0.6911 Epoch 27/250 50/50 [==============================] - 1s 13ms/step - loss: 2.3987 - accuracy: 0.9334 - val_loss: 2.6478 - val_accuracy: 0.6911 Epoch 28/250 50/50 [==============================] - 1s 13ms/step - loss: 2.4009 - accuracy: 0.9284 - val_loss: 2.6439 - val_accuracy: 0.6852
plot_accuracy(history=history)
plot_loss(history=history)
plot_confussion_matrix2(model, X_test, y_test,ID_LABEL_MAP.values())
predictions_probs = model.predict(X_test)
y_pred = np.argmax(model.predict(X_test), axis=1)
i = 0
#Correct prediction:
while y_test[i] != y_pred[i]:
i+=1
show_image(ID_LABEL_MAP[y_pred[i]],X_test[i])
print("Probabilities of correctly classified image:",predictions_probs[i])
#Incorrect prediction:
while y_test[i] == y_pred[i]:
i+=1
show_image(ID_LABEL_MAP[y_pred[i]],X_test[i])
print("Probabilities of incorrectly classified image:",predictions_probs[i])
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Probabilities of correctly classified image: [1.5788127e-06 9.9670249e-01 8.3686855e-06 3.7532590e-08 2.7730252e-06 1.1373787e-10 4.8285335e-07 5.2656524e-09 1.1143637e-03 8.8228636e-10 7.0673536e-07 1.0101077e-06 2.9500674e-07 1.5254402e-09 3.3826273e-05 1.1074154e-03 2.4261467e-06 2.6279577e-06 2.4534790e-08 9.7327499e-04 1.6775035e-06 3.2505704e-10 1.7268712e-07 4.6058380e-05 3.1993042e-07 5.7360086e-08]
Probabilities of incorrectly classified image: [8.5776541e-09 6.8765664e-03 2.2107688e-06 8.3377682e-02 8.1283522e-01 1.2101970e-08 5.5161930e-08 4.0924101e-07 9.5178337e-05 2.0096518e-10 7.9234997e-07 1.3349979e-04 1.3897384e-08 7.1034421e-08 1.2160309e-03 1.7609487e-06 4.3995102e-04 9.7224035e-04 5.3661262e-07 9.0749733e-02 1.2173884e-07 5.9128374e-10 1.2603628e-07 1.3682507e-05 3.2798534e-03 4.2254774e-06]
n = len(LABEL_ID_MAP.keys())
confussion_matrix = tf.math.confusion_matrix(y_test,y_pred)
cm = np.array(confussion_matrix)
recall = np.diag(cm) / np.sum(cm, axis = 1)
precision = np.diag(cm) / np.sum(cm, axis = 0)
print("Label", "Recall", "Precision")
for class_id in range(0,n):
print(ID_LABEL_MAP[class_id], recall[class_id], precision[class_id])
Label Recall Precision helicopter 0.7692307692307693 0.6451612903225806 ewer 0.5769230769230769 0.5 ketch 0.8148148148148148 0.7857142857142857 Faces_easy 1.0 0.7714285714285715 Faces 0.7037037037037037 0.95 scorpion 0.28 0.875 Leopards 0.9259259259259259 0.9615384615384616 watch 0.7037037037037037 0.8260869565217391 hawksbill 0.8888888888888888 0.48 Motorbikes 0.8518518518518519 1.0 menorah 0.6538461538461539 0.9444444444444444 revolver 0.8 0.5405405405405406 bonsai 0.4074074074074074 0.6875 airplanes 0.9629629629629629 0.8387096774193549 starfish 0.19230769230769232 1.0 buddha 0.5769230769230769 0.625 ibis 0.6666666666666666 0.5925925925925926 brain 0.6666666666666666 0.6206896551724138 trilobite 0.8846153846153846 0.92 kangaroo 0.8076923076923077 0.29577464788732394 chandelier 0.48148148148148145 0.7647058823529411 laptop 0.0 nan sunflower 0.76 0.7037037037037037 car_side 0.9629629629629629 0.9285714285714286 butterfly 0.48148148148148145 0.5909090909090909 grand_piano 0.8888888888888888 0.75
/tmp/ipykernel_113926/524993589.py:5: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
Our neural network almost always has problem with 1 or 2 classes. Sometimes its ewer, somentimes buddha or both. This time it appears that laptops have beated it. At this stage of the experiment, we suspect it might due to the structure of the neural network. But what's interesting, it doesn't matter on how many classes we train the network if the number is bigger than around 25 classes.
def save_model(model):
with open('my_model.json', "w+") as output_file:
output_file.write(model.to_json())
model.save_weights("my_weights.h5")
def load_model(path_to_model ='my_model.json', path_to_weights="my_weights.h5"):
custom_objects = {"CNNBlock": CNNBlock}
with open('my_model.json', "r") as output_file:
with keras.utils.custom_object_scope(custom_objects):
model = keras.models.model_from_json(output_file.read())
model.load_weights(path_to_weights)
return model
save_model(model)
loaded_model =load_model()
loaded_model.compile(optimizer="adam",
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=["accuracy"])
loss, acc = loaded_model.evaluate(X_test, y_test, verbose=0)
print(f"Val accuracy: {acc}")
plot_confussion_matrix2(loaded_model, X_test, y_test,ID_LABEL_MAP.values())
Val accuracy: 0.6852123141288757
Last validation accuracy is the same. Confusion matrices look the same.
# Plot accuracies
def plot_accuracies(histories):
fig, axes = plt.subplots(1, len(histories))
fig.set_size_inches(20, 8)
counter = 0
for history in histories:
axes[counter].plot(history.history['accuracy'], label="accuracy",)
axes[counter].plot(history.history['val_accuracy'], label = "val_accuracy")
plt.yticks(np.linspace(0, 1, num=11))
axes[counter].axhline(y=np.max(history.history['val_accuracy']), color='g', linestyle='--', label="Best accuracy")
counter +=1
plt.legend()
plt.show()
def plot_losses(histories):
fig, axes = plt.subplots(1, len(histories))
fig.set_size_inches(20, 8)
counter = 0
for history in histories:
axes[counter].plot(history.history['loss'], label="loss",)
axes[counter].plot(history.history['val_loss'], label = "loss")
counter +=1
plt.legend()
plt.show()
def plot_cm_matrixes(models, X_tests, y_tests):
fig, axes = plt.subplots(1, len(models))
fig.set_size_inches(20, 8)
counter = 0
for model__ in models:
# logits = model.predict(X_tests[counter])
y_pred = np.argmax(model__.predict(X_tests[counter]), axis=1)
confussion_matrix = tf.math.confusion_matrix(y_tests[counter], y_pred)
axes[counter].imshow(confussion_matrix)
counter += 1
# plt.legend()
plt.show()
def plot_pretty_cm_matrixes(models, names, X_tests, y_tests, ID_LABEL_MAPS):
counter = 0
for model__ in models:
plot_confussion_matrix2(model__, X_tests[counter], y_tests[counter],ID_LABEL_MAPS[counter].values(),
title="Model name: " + names[counter])
counter +=1
def recall_precision(models, names, X_tests, y_tests, ID_LABELS_MAPS):
counter =0
for model__ in models:
# logits = model.predict(X_test)
y_pred = np.argmax(model__.predict(X_tests[counter]), axis=1)
confussion_matrix = tf.math.confusion_matrix(y_tests[counter],y_pred)
cm = np.array(confussion_matrix)
recall = np.diag(cm) / np.sum(cm, axis = 1)
precision = np.diag(cm) / np.sum(cm, axis = 0)
print(str(names[counter]), "Label", "Recall", "Precision")
print("Worst recall for: ", ID_LABELS_MAPS[counter][np.argmin(recall)], recall[np.argmin(recall)])
print("Worst precision for: ", ID_LABELS_MAPS[counter][np.argmin(precision)], recall[np.argmin(precision)] )
for class_id in range(0,len(ID_LABELS_MAPS[counter])):
print(ID_LABELS_MAPS[counter][class_id], recall[class_id], precision[class_id])
counter += 1
print()
def compare_models(models, names, img_process_funcs=None, iter_data=False, num_epochs=250, batch_size=32,
number_of_classes_to_pick=None, same_data_for_every_process_func=False):
assert len(models) == len(names)
# print(''.join([model_name +": " + str(model.count_params()) + " " for model_, model_name in zip(models, names)]))
training_times = []
histories = []
X_tests = []
y_tests = []
real_models = []
ID_LABELS_MAPS = []
#Loads data
if not iter_data and not same_data_for_every_process_func:
assert same_data_for_every_process_func is False
print("--- I'm not iterating data, same data for very model. ---")
labels, files = select_images(Path("./src/data/Caltech-101"), down_sample=80,num_of_classes=26)
LABEL_ID_MAP = {label: i for i, label in enumerate(labels)}
ID_LABEL_MAP = {i: label for i, label in enumerate(labels)}
CLASS_SIZE = {label: len(files[i]) for i, label in enumerate(labels)}
labels, not_normalized_images = load_images(labels, files)
standardized_images = standardize_images(not_normalized_images)
y = np.array([LABEL_ID_MAP[label] for label in labels])
X_train, X_test, y_train, y_test = train_test_split(standardized_images, y, stratify=y, test_size=0.3, random_state=12)
for __ in range(0, len(models)):
X_tests.append(X_test)
y_tests.append(y_test)
ID_LABELS_MAPS.append(ID_LABEL_MAP)
if same_data_for_every_process_func:
assert iter_data is False
print("--- I'm not iterating data, same data for very model. ---")
labels, files = select_images(Path("./src/data/Caltech-101"), down_sample=80,num_of_classes=26)
LABEL_ID_MAP = {label: i for i, label in enumerate(labels)}
ID_LABEL_MAP = {i: label for i, label in enumerate(labels)}
CLASS_SIZE = {label: len(files[i]) for i, label in enumerate(labels)}
labels, not_normalized_images = load_images(labels, files)
standardized_images = standardize_images(not_normalized_images)
y = np.array([LABEL_ID_MAP[label] for label in labels])
X_train, X_test, y_train, y_test = train_test_split(standardized_images, y, stratify=y, test_size=0.3,
random_state=12)
for __ in range(0, len(models)):
ID_LABELS_MAPS.append(ID_LABEL_MAP)
print("Number of trainable params:")
counter = 0
for model in models:
print("Model name: ",names[counter])
if iter_data:
assert img_process_funcs is not None
assert number_of_classes_to_pick is not None
assert len(models) == len(names) == len(img_process_funcs) == len(number_of_classes_to_pick)
labels, files = select_images(Path("./src/data/Caltech-101"), down_sample=80,
num_of_classes=number_of_classes_to_pick[counter])
LABEL_ID_MAP = {label: i for i, label in enumerate(labels)}
ID_LABEL_MAP = {i: label for i, label in enumerate(labels)}
CLASS_SIZE = {label: len(files[i]) for i, label in enumerate(labels)}
labels, not_normalized_images = load_images(labels, files)
standardized_images = img_process_funcs[counter](not_normalized_images)
y = np.array([LABEL_ID_MAP[label] for label in labels])
X_train, X_test, y_train, y_test = train_test_split(standardized_images, y, stratify=y, test_size=0.3)
print(F"Loading data. Number of classes: {number_of_classes_to_pick[counter]} Standarazing function: {img_process_funcs[counter]}")
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)
ID_LABELS_MAPS.append(ID_LABEL_MAP)
X_tests.append(X_test)
y_tests.append(y_test)
if same_data_for_every_process_func:
assert iter_data is False
standardized_images = img_process_funcs[counter](not_normalized_images)
X_train, X_test, y_train, y_test = train_test_split(standardized_images, y, stratify=y, test_size=0.3,
random_state=12)
X_tests.append(X_test)
y_tests.append(y_test)
model_ = create_model(len(ID_LABEL_MAP),model["filter_numbers"], model["filter_sizes"],
model["activations"], model["dropout"],
model["pool_sizes"], model["dense_sizes"], model["dense_activations"])
real_models.append(model_)
model_.compile(optimizer="adam",
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=["accuracy"])
check_time = time.time()
history = model_.fit(X_train, y_train,
epochs=num_epochs,
validation_data=(X_test, y_test),
batch_size=batch_size,
callbacks=[
keras.callbacks.EarlyStopping(
monitor="val_accuracy",
patience=6)], verbose=0) #verbose=0 disables logging
training_times.append(time.time()-check_time)
histories.append(history)
print("Number of params: "+ str(model_.count_params()))
print()
counter +=1
print("Traning times: ")
print(''.join([model_name +": " + str(training_time) + " " for training_time, model_name in zip(training_times, names)]))
print("Best val accuracies: ")
print(''.join([model_name +": " + str(np.max(history.history['val_accuracy'])) + "\n" for history, model_name in zip(histories, names)]))
plot_accuracies(histories)
plot_losses(histories)
# plot_cm_matrixes(real_models, X_tests, y_tests)
recall_precision(real_models, names, X_tests, y_tests, ID_LABELS_MAPS)
plot_pretty_cm_matrixes(real_models, names, X_tests, y_tests, ID_LABELS_MAPS)
To compare the networks, we created this scary looking function. With it, one can test every adjastubel hyper parameter of our model. (Number of filters, filter sizes, activation functions, dropouts, pool_sizes, number of blocks, dense layer structure).
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
dropout = [0.2]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_sizes = [256, 100]
dense_activations = ["sigmoid", "relu"]
num_of_classes = [2,3,5,10,15,20,26]
models = []
names = []
img_process_funcs = []
for num_class in num_of_classes:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"Model with {str(num_class)} classes.")
img_process_funcs.append(standardize_images)
compare_models(models=models, names=names, iter_data=True,
img_process_funcs=img_process_funcs, number_of_classes_to_pick=num_of_classes)
#Decided to work o 26 classes so it wont be too easy
Number of trainable params: Model name: Model with 2 classes. Loading data. Number of classes: 2 Standarazing function: <function standardize_images at 0x7fa87eeb2280> (126, 64, 64, 3) (126,) (54, 64, 64, 3) (54,) Number of params: 570222 Model name: Model with 3 classes. Loading data. Number of classes: 3 Standarazing function: <function standardize_images at 0x7fa87eeb2280> (189, 64, 64, 3) (189,) (81, 64, 64, 3) (81,) Number of params: 570323 Model name: Model with 5 classes. Loading data. Number of classes: 5 Standarazing function: <function standardize_images at 0x7fa87eeb2280> (315, 64, 64, 3) (315,) (135, 64, 64, 3) (135,) Number of params: 570525 Model name: Model with 10 classes. Loading data. Number of classes: 10 Standarazing function: <function standardize_images at 0x7fa87eeb2280> (630, 64, 64, 3) (630,) (270, 64, 64, 3) (270,) Number of params: 571030 Model name: Model with 15 classes. Loading data. Number of classes: 15 Standarazing function: <function standardize_images at 0x7fa87eeb2280> (943, 64, 64, 3) (943,) (405, 64, 64, 3) (405,) Number of params: 571535 Model name: Model with 20 classes. Loading data. Number of classes: 20 Standarazing function: <function standardize_images at 0x7fa87eeb2280> (1244, 64, 64, 3) (1244,) (534, 64, 64, 3) (534,) Number of params: 572040 Model name: Model with 26 classes. Loading data. Number of classes: 26 Standarazing function: <function standardize_images at 0x7fa87eeb2280> (1592, 64, 64, 3) (1592,) (683, 64, 64, 3) (683,) Number of params: 572646 Traning times: Model with 2 classes.: 1.9498491287231445 Model with 3 classes.: 1.5570287704467773 Model with 5 classes.: 1.91945219039917 Model with 10 classes.: 8.894641637802124 Model with 15 classes.: 11.582173109054565 Model with 20 classes.: 24.678088903427124 Model with 26 classes.: 23.69823122024536 Best val accuracies: Model with 2 classes.: 1.0 Model with 3 classes.: 0.9876543283462524 Model with 5 classes.: 0.7777777910232544 Model with 10 classes.: 0.9222221970558167 Model with 15 classes.: 0.8493826985359192 Model with 20 classes.: 0.8389512896537781 Model with 26 classes.: 0.7672035098075867
Model with 2 classes. Label Recall Precision Worst recall for: Motorbikes 0.9629629629629629 Worst precision for: airplanes 1.0 airplanes 1.0 0.9642857142857143 Motorbikes 0.9629629629629629 1.0 Model with 3 classes. Label Recall Precision Worst recall for: Motorbikes 0.7037037037037037 Worst precision for: airplanes 1.0 airplanes 1.0 0.675 Motorbikes 0.7037037037037037 1.0 Faces_easy 0.8148148148148148 1.0 Model with 5 classes. Label Recall Precision Worst recall for: watch 0.1111111111111111 Worst precision for: airplanes 1.0 airplanes 1.0 0.3068181818181818 Motorbikes 0.2962962962962963 1.0 Faces_easy 0.8888888888888888 0.75 Faces 0.14814814814814814 1.0 watch 0.1111111111111111 1.0 Model with 10 classes. Label Recall Precision Worst recall for: Motorbikes 0.8518518518518519 Worst precision for: ketch 0.9259259259259259 airplanes 0.9629629629629629 0.8125 Motorbikes 0.8518518518518519 1.0 Faces_easy 0.9629629629629629 0.896551724137931 Faces 0.9259259259259259 0.9259259259259259 watch 0.8888888888888888 0.96 Leopards 1.0 1.0 bonsai 0.8518518518518519 1.0 car_side 1.0 1.0 ketch 0.9259259259259259 0.8064516129032258 chandelier 0.8518518518518519 0.8846153846153846 Model with 15 classes. Label Recall Precision Worst recall for: hawksbill 0.6666666666666666 Worst precision for: butterfly 0.7777777777777778 airplanes 0.9629629629629629 0.8387096774193549 Motorbikes 1.0 0.9642857142857143 Faces_easy 0.8888888888888888 0.8275862068965517 Faces 0.7037037037037037 0.8636363636363636 watch 0.7777777777777778 0.84 Leopards 1.0 0.9642857142857143 bonsai 0.8148148148148148 0.88 car_side 0.9259259259259259 0.8928571428571429 ketch 0.9629629629629629 0.9285714285714286 chandelier 0.7777777777777778 0.8076923076923077 hawksbill 0.6666666666666666 0.6923076923076923 grand_piano 0.8518518518518519 0.92 brain 0.8518518518518519 0.8846153846153846 butterfly 0.7777777777777778 0.6363636363636364 helicopter 0.7407407407407407 0.8 Model with 20 classes. Label Recall Precision Worst recall for: butterfly 0.4444444444444444 Worst precision for: kangaroo 0.9615384615384616 airplanes 0.9259259259259259 0.9615384615384616 Motorbikes 1.0 0.9642857142857143 Faces_easy 1.0 1.0 Faces 0.9259259259259259 0.8064516129032258 watch 0.8148148148148148 0.8148148148148148 Leopards 0.9259259259259259 0.9259259259259259 bonsai 0.7777777777777778 0.75 car_side 1.0 0.9 ketch 0.9259259259259259 0.9259259259259259 chandelier 0.7777777777777778 0.9130434782608695 hawksbill 0.6666666666666666 0.782608695652174 grand_piano 0.7777777777777778 1.0 brain 0.7777777777777778 0.8076923076923077 butterfly 0.4444444444444444 0.8571428571428571 helicopter 0.7307692307692307 0.6785714285714286 menorah 0.7692307692307693 0.8695652173913043 starfish 0.8461538461538461 0.7096774193548387 trilobite 0.9230769230769231 1.0 kangaroo 0.9615384615384616 0.5813953488372093 ewer 0.7307692307692307 0.7037037037037037 Model with 26 classes. Label Recall Precision Worst recall for: kangaroo 0.38461538461538464 Worst precision for: helicopter 0.6923076923076923 airplanes 0.7777777777777778 1.0 Motorbikes 0.7777777777777778 0.9130434782608695 Faces_easy 1.0 1.0 Faces 0.9259259259259259 1.0 watch 0.7407407407407407 0.9090909090909091 Leopards 0.9259259259259259 0.8620689655172413 bonsai 0.8888888888888888 0.8 car_side 0.8888888888888888 0.8 ketch 0.8518518518518519 0.696969696969697 chandelier 0.7777777777777778 0.84 hawksbill 0.5555555555555556 0.8823529411764706 grand_piano 0.6666666666666666 1.0 brain 0.7777777777777778 0.7 butterfly 0.5925925925925926 0.5517241379310345 helicopter 0.6923076923076923 0.375 menorah 0.6538461538461539 0.9444444444444444 starfish 0.7307692307692307 0.40425531914893614 trilobite 1.0 0.9629629629629629 kangaroo 0.38461538461538464 0.7142857142857143 ewer 0.76 0.7307692307692307 buddha 0.46153846153846156 0.6 sunflower 0.7692307692307693 0.8695652173913043 scorpion 0.52 0.41935483870967744 revolver 0.96 0.7058823529411765 laptop 0.7083333333333334 0.7083333333333334 ibis 0.4166666666666667 0.8333333333333334
The model worked best with 2-class classification problem, which is not that surprising. What is surprising is that the 3rd worst performance was achievied when classyfing just 5 classes. After further inspection, we think it was caused by inclusion of sometimes difficult to recognize for the network class - the watch, although next models didn't have a problem with it. What is more, validation accuracy increases, but then suddenly drops with each consecutive epoch. It seems that the model started to extremely overfit, similarly as in 3 class case, but data for first epochs was chosen more luckily in that model. Other learning curves seem normal and not that interesting.
As it was expected, the more classes included, the longer the time to train the network on average, and the lower validation accuracy.
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
dropout = [0.2]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_activations = ["sigmoid", "relu"]
denses_sizeses = [[256, 256], [100, 256],[256, 100] ,[100, 100], [50,50], [500, 500], [20,100]]
models = []
names = []
for dense_sizes in denses_sizeses:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{str(dense_sizes)}")
compare_models(models=models, names=names)
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: [256, 256] Number of params: 616794 Model name: [100, 256] Number of params: 257214 Model name: [256, 100] Number of params: 572646 Model name: [100, 100] Number of params: 237402 Model name: [50, 50] Number of params: 126102 Model name: [500, 500] Number of params: 1307802 Model name: [20, 100] Number of params: 65482 Traning times: [256, 256]: 24.290355443954468 [100, 256]: 18.77979826927185 [256, 100]: 16.03062129020691 [100, 100]: 17.104461908340454 [50, 50]: 32.739362478256226 [500, 500]: 33.848774671554565 [20, 100]: 22.762860774993896 Best val accuracies: [256, 256]: 0.6969253420829773 [100, 256]: 0.7364568114280701 [256, 100]: 0.664714515209198 [100, 100]: 0.6881405711174011 [50, 50]: 0.6910688281059265 [500, 500]: 0.7423133254051208 [20, 100]: 0.6486091017723083
/tmp/ipykernel_113926/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
[256, 256] Label Recall Precision Worst recall for: menorah 0.0 Worst precision for: menorah 0.0 airplanes 1.0 0.75 Motorbikes 0.9629629629629629 0.9285714285714286 Faces_easy 1.0 0.9310344827586207 Faces 0.8518518518518519 0.6216216216216216 watch 0.6666666666666666 0.782608695652174 Leopards 0.9629629629629629 0.8387096774193549 bonsai 0.6666666666666666 0.5806451612903226 car_side 1.0 0.8709677419354839 ketch 0.8888888888888888 0.7741935483870968 chandelier 0.7407407407407407 0.625 hawksbill 0.6666666666666666 0.6 grand_piano 0.7777777777777778 0.9545454545454546 brain 0.7777777777777778 0.7241379310344828 butterfly 0.48148148148148145 0.7647058823529411 helicopter 0.5384615384615384 0.6086956521739131 menorah 0.0 nan starfish 0.6153846153846154 0.37209302325581395 trilobite 0.9615384615384616 0.8064516129032258 kangaroo 0.7307692307692307 0.475 ewer 0.5769230769230769 0.5555555555555556 buddha 0.6923076923076923 0.6 sunflower 0.8 0.9090909090909091 scorpion 0.6 0.40540540540540543 revolver 0.8 0.8695652173913043 laptop 0.0 nan ibis 0.0 nan [100, 256] Label Recall Precision Worst recall for: ibis 0.0 Worst precision for: ibis 0.0 airplanes 0.9259259259259259 0.8333333333333334 Motorbikes 1.0 0.9310344827586207 Faces_easy 0.9629629629629629 0.9285714285714286 Faces 0.8148148148148148 1.0 watch 0.7037037037037037 0.5428571428571428 Leopards 0.7407407407407407 1.0 bonsai 0.5555555555555556 0.9375 car_side 1.0 0.9310344827586207 ketch 0.8518518518518519 0.8846153846153846 chandelier 0.8518518518518519 0.4423076923076923 hawksbill 0.7037037037037037 0.7037037037037037 grand_piano 0.7407407407407407 1.0 brain 0.7407407407407407 0.625 butterfly 0.48148148148148145 0.6842105263157895 helicopter 0.5 0.8125 menorah 0.9230769230769231 0.8275862068965517 starfish 0.6153846153846154 0.5714285714285714 trilobite 0.9230769230769231 0.96 kangaroo 0.6538461538461539 0.7083333333333334 ewer 0.5384615384615384 0.4666666666666667 buddha 0.5384615384615384 0.875 sunflower 0.8 0.9523809523809523 scorpion 0.64 0.3018867924528302 revolver 0.88 0.7857142857142857 laptop 0.875 0.75 ibis 0.0 nan [256, 100] Label Recall Precision Worst recall for: Faces 0.0 Worst precision for: Faces 0.0 airplanes 0.8148148148148148 0.7333333333333333 Motorbikes 1.0 0.8709677419354839 Faces_easy 1.0 0.8181818181818182 Faces 0.0 nan watch 0.8148148148148148 0.8148148148148148 Leopards 1.0 0.8181818181818182 bonsai 0.6296296296296297 0.85 car_side 1.0 0.7714285714285715 ketch 0.9629629629629629 0.6341463414634146 chandelier 0.6666666666666666 0.6666666666666666 hawksbill 0.7777777777777778 0.7777777777777778 grand_piano 0.7777777777777778 0.7777777777777778 brain 0.8148148148148148 0.6470588235294118 butterfly 0.0 nan helicopter 0.38461538461538464 0.625 menorah 0.7692307692307693 0.7407407407407407 starfish 0.7307692307692307 0.3584905660377358 trilobite 0.9230769230769231 0.8571428571428571 kangaroo 0.6923076923076923 0.375 ewer 0.5769230769230769 0.4411764705882353 buddha 0.3076923076923077 0.42105263157894735 sunflower 0.84 0.9545454545454546 scorpion 0.44 0.3235294117647059 revolver 0.84 0.5675675675675675 laptop 0.0 nan ibis 0.0 nan [100, 100] Label Recall Precision Worst recall for: helicopter 0.0 Worst precision for: helicopter 0.0 airplanes 0.8518518518518519 0.6764705882352942 Motorbikes 1.0 0.9 Faces_easy 0.9629629629629629 0.9285714285714286 Faces 0.8518518518518519 0.8214285714285714 watch 0.7037037037037037 0.9047619047619048 Leopards 0.8888888888888888 0.8888888888888888 bonsai 0.6666666666666666 0.5625 car_side 1.0 0.7941176470588235 ketch 0.9629629629629629 0.49056603773584906 chandelier 0.7407407407407407 0.6896551724137931 hawksbill 0.7777777777777778 0.7 grand_piano 0.7407407407407407 0.8333333333333334 brain 0.7777777777777778 0.6176470588235294 butterfly 0.5925925925925926 0.38095238095238093 helicopter 0.0 nan menorah 0.8846153846153846 0.7931034482758621 starfish 0.6538461538461539 0.53125 trilobite 0.9615384615384616 0.9259259259259259 kangaroo 0.6923076923076923 0.43902439024390244 ewer 0.0 nan buddha 0.0 nan sunflower 0.8 0.8695652173913043 scorpion 0.56 0.5384615384615384 revolver 0.8 0.7142857142857143 laptop 0.875 0.6774193548387096 ibis 0.0 nan [50, 50] Label Recall Precision Worst recall for: menorah 0.0 Worst precision for: menorah 0.0 airplanes 0.8518518518518519 0.92 Motorbikes 0.9629629629629629 0.896551724137931 Faces_easy 0.9629629629629629 0.8387096774193549 Faces 0.7407407407407407 0.8695652173913043 watch 0.6666666666666666 0.6 Leopards 0.9629629629629629 0.8666666666666667 bonsai 0.6296296296296297 0.7727272727272727 car_side 0.8888888888888888 0.8888888888888888 ketch 0.7407407407407407 0.6896551724137931 chandelier 0.6296296296296297 0.5 hawksbill 0.7407407407407407 0.6060606060606061 grand_piano 0.7407407407407407 0.9523809523809523 brain 0.5925925925925926 0.6956521739130435 butterfly 0.5555555555555556 0.5357142857142857 helicopter 0.7307692307692307 0.40425531914893614 menorah 0.0 nan starfish 0.0 nan trilobite 1.0 0.896551724137931 kangaroo 0.7307692307692307 0.37254901960784315 ewer 0.5384615384615384 0.4666666666666667 buddha 0.5384615384615384 0.5384615384615384 sunflower 0.68 1.0 scorpion 0.44 0.39285714285714285 revolver 0.76 0.7916666666666666 laptop 0.7083333333333334 0.8947368421052632 ibis 0.5833333333333334 0.5185185185185185 [500, 500] Label Recall Precision Worst recall for: helicopter 0.34615384615384615 Worst precision for: scorpion 0.52 airplanes 0.7777777777777778 0.75 Motorbikes 0.9629629629629629 0.7647058823529411 Faces_easy 0.9629629629629629 1.0 Faces 0.9259259259259259 0.8333333333333334 watch 0.7037037037037037 0.76 Leopards 1.0 0.7714285714285715 bonsai 0.6296296296296297 0.68 car_side 0.9629629629629629 0.8125 ketch 0.8888888888888888 0.96 chandelier 0.6296296296296297 0.5862068965517241 hawksbill 0.6666666666666666 0.72 grand_piano 0.7777777777777778 1.0 brain 0.7407407407407407 0.6060606060606061 butterfly 0.48148148148148145 0.6190476190476191 helicopter 0.34615384615384615 0.5 menorah 0.6923076923076923 0.9 starfish 0.6153846153846154 0.48484848484848486 trilobite 0.9230769230769231 0.9230769230769231 kangaroo 0.7307692307692307 0.59375 ewer 0.5 0.65 buddha 0.5384615384615384 0.56 sunflower 0.64 0.9411764705882353 scorpion 0.52 0.4642857142857143 revolver 0.84 0.8076923076923077 laptop 0.75 0.5454545454545454 ibis 0.625 0.9375 [20, 100] Label Recall Precision Worst recall for: butterfly 0.0 Worst precision for: butterfly 0.0 airplanes 0.8888888888888888 0.8275862068965517 Motorbikes 0.9629629629629629 0.8666666666666667 Faces_easy 0.9629629629629629 0.9629629629629629 Faces 0.8518518518518519 0.8214285714285714 watch 0.7037037037037037 0.8260869565217391 Leopards 0.8518518518518519 0.92 bonsai 0.5185185185185185 0.7 car_side 0.9629629629629629 0.7428571428571429 ketch 0.7407407407407407 0.7142857142857143 chandelier 0.6666666666666666 0.47368421052631576 hawksbill 0.7407407407407407 0.5 grand_piano 0.6666666666666666 0.6923076923076923 brain 0.6296296296296297 0.7391304347826086 butterfly 0.0 nan helicopter 0.6538461538461539 0.3695652173913043 menorah 0.6923076923076923 0.8571428571428571 starfish 0.0 nan trilobite 0.9230769230769231 0.8275862068965517 kangaroo 0.6538461538461539 0.425 ewer 0.6923076923076923 0.35294117647058826 buddha 0.0 nan sunflower 0.72 0.8571428571428571 scorpion 0.56 0.4666666666666667 revolver 0.84 0.65625 laptop 0.0 nan ibis 0.625 0.36585365853658536
To tinker a little bit with the structure of the network, we decided to test a little bit more the structure of dense layers. How much number of neurons in that layers matter? Can there be too litle of them?. The test showed, that 500,500 structure seemed to be the best, but we choose [100,256], because it has much less parameters and it traning time was two times faster. Much simpler, but almost as good.
To be sure we searched once again in closer neighbourhood of best result:
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
dropout = [0.2]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_activations = ["sigmoid", "relu"]
denses_sizeses = [[128,256],[128,200],[80,256], [80,200], [100,200]]
models = []
names = []
for dense_sizes in denses_sizeses:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{str(dense_sizes)}")
compare_models(models=models, names=names)
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: [128, 256] Number of params: 321754 Model name: [128, 200] Number of params: 313074 Model name: [80, 256] Number of params: 211114 Model name: [80, 200] Number of params: 205122 Model name: [100, 200] Number of params: 250102 Traning times: [128, 256]: 20.4942524433136 [128, 200]: 22.86384916305542 [80, 256]: 17.626417636871338 [80, 200]: 22.796733856201172 [100, 200]: 14.16812801361084 Best val accuracies: [128, 256]: 0.6954612135887146 [128, 200]: 0.7423133254051208 [80, 256]: 0.7247437834739685 [80, 200]: 0.7130307555198669 [100, 200]: 0.6720351576805115
/tmp/ipykernel_113926/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
[128, 256] Label Recall Precision Worst recall for: watch 0.0 Worst precision for: watch 0.0 airplanes 0.9629629629629629 0.8666666666666667 Motorbikes 0.8518518518518519 0.8846153846153846 Faces_easy 1.0 0.9642857142857143 Faces 0.8148148148148148 0.9166666666666666 watch 0.0 nan Leopards 0.6666666666666666 0.9473684210526315 bonsai 0.9259259259259259 0.7575757575757576 car_side 0.9629629629629629 0.9629629629629629 ketch 0.7777777777777778 0.8076923076923077 chandelier 0.7407407407407407 0.5882352941176471 hawksbill 0.5555555555555556 0.6 grand_piano 0.8148148148148148 0.8148148148148148 brain 0.7037037037037037 0.8636363636363636 butterfly 0.5185185185185185 0.7368421052631579 helicopter 0.34615384615384615 0.5625 menorah 0.6538461538461539 0.8947368421052632 starfish 0.0 nan trilobite 0.8846153846153846 0.9583333333333334 kangaroo 0.5 0.5 ewer 0.6923076923076923 0.42857142857142855 buddha 0.6538461538461539 0.6071428571428571 sunflower 0.84 0.84 scorpion 0.32 0.42105263157894735 revolver 0.88 0.88 laptop 0.6666666666666666 0.8888888888888888 ibis 0.875 0.2079207920792079 [128, 200] Label Recall Precision Worst recall for: watch 0.0 Worst precision for: watch 0.0 airplanes 0.8888888888888888 0.8571428571428571 Motorbikes 0.9259259259259259 0.8333333333333334 Faces_easy 1.0 1.0 Faces 0.8518518518518519 0.92 watch 0.0 nan Leopards 0.8888888888888888 0.9230769230769231 bonsai 0.8888888888888888 0.8571428571428571 car_side 0.9629629629629629 0.8666666666666667 ketch 0.8148148148148148 0.88 chandelier 0.7037037037037037 0.5428571428571428 hawksbill 0.5185185185185185 0.6086956521739131 grand_piano 0.8148148148148148 1.0 brain 0.8148148148148148 0.7333333333333333 butterfly 0.4444444444444444 0.6 helicopter 0.38461538461538464 0.47619047619047616 menorah 0.6538461538461539 0.85 starfish 0.6538461538461539 0.4594594594594595 trilobite 0.9230769230769231 0.8571428571428571 kangaroo 0.5769230769230769 0.6 ewer 0.7692307692307693 0.5128205128205128 buddha 0.6153846153846154 0.36363636363636365 sunflower 0.92 0.9583333333333334 scorpion 0.48 0.46153846153846156 revolver 0.88 0.9565217391304348 laptop 0.5416666666666666 0.7647058823529411 ibis 0.6666666666666666 0.5333333333333333 [80, 256] Label Recall Precision Worst recall for: butterfly 0.0 Worst precision for: butterfly 0.0 airplanes 0.8888888888888888 0.96 Motorbikes 0.9259259259259259 0.8928571428571429 Faces_easy 1.0 1.0 Faces 0.8148148148148148 0.9166666666666666 watch 0.6296296296296297 0.85 Leopards 0.8888888888888888 0.96 bonsai 0.7037037037037037 0.76 car_side 0.9629629629629629 0.9285714285714286 ketch 0.9259259259259259 0.8064516129032258 chandelier 0.7407407407407407 0.5128205128205128 hawksbill 0.8148148148148148 0.3728813559322034 grand_piano 0.7777777777777778 1.0 brain 0.6666666666666666 0.8571428571428571 butterfly 0.0 nan helicopter 0.5769230769230769 0.625 menorah 0.7692307692307693 0.7142857142857143 starfish 0.5 0.5416666666666666 trilobite 0.9615384615384616 0.9615384615384616 kangaroo 0.6923076923076923 0.45 ewer 0.0 nan buddha 0.5769230769230769 0.6521739130434783 sunflower 0.88 0.9565217391304348 scorpion 0.6 0.3488372093023256 revolver 0.92 0.8846153846153846 laptop 0.6666666666666666 0.7272727272727273 ibis 0.6666666666666666 0.5161290322580645 [80, 200] Label Recall Precision Worst recall for: brain 0.0 Worst precision for: brain 0.0 airplanes 0.9259259259259259 1.0 Motorbikes 0.9629629629629629 0.9629629629629629 Faces_easy 1.0 0.9310344827586207 Faces 0.7777777777777778 0.875 watch 0.7407407407407407 0.8 Leopards 0.8888888888888888 0.9230769230769231 bonsai 0.7777777777777778 0.6774193548387096 car_side 1.0 0.7941176470588235 ketch 0.9259259259259259 0.8333333333333334 chandelier 0.9259259259259259 0.5952380952380952 hawksbill 0.5555555555555556 0.4838709677419355 grand_piano 0.7777777777777778 0.9545454545454546 brain 0.0 nan butterfly 0.5555555555555556 0.4411764705882353 helicopter 0.5384615384615384 0.5 menorah 0.7692307692307693 0.9090909090909091 starfish 0.0 nan trilobite 0.9230769230769231 0.8888888888888888 kangaroo 0.6153846153846154 0.48484848484848486 ewer 0.6923076923076923 0.45 buddha 0.46153846153846156 0.8 sunflower 0.76 1.0 scorpion 0.52 0.3333333333333333 revolver 0.92 0.92 laptop 0.6666666666666666 0.7619047619047619 ibis 0.625 0.4411764705882353 [100, 200] Label Recall Precision Worst recall for: helicopter 0.0 Worst precision for: helicopter 0.0 airplanes 0.9259259259259259 0.625 Motorbikes 0.8888888888888888 0.8571428571428571 Faces_easy 1.0 1.0 Faces 0.8518518518518519 0.7666666666666667 watch 0.7777777777777778 0.7 Leopards 0.5185185185185185 0.875 bonsai 0.7777777777777778 0.8076923076923077 car_side 1.0 0.6585365853658537 ketch 0.9629629629629629 0.6190476190476191 chandelier 0.6666666666666666 0.5806451612903226 hawksbill 0.5925925925925926 0.7619047619047619 grand_piano 0.8518518518518519 0.6216216216216216 brain 0.6296296296296297 0.8095238095238095 butterfly 0.3333333333333333 0.6 helicopter 0.0 nan menorah 0.6538461538461539 0.9444444444444444 starfish 0.4230769230769231 0.6875 trilobite 0.8076923076923077 1.0 kangaroo 0.34615384615384615 0.45 ewer 0.6923076923076923 0.391304347826087 buddha 0.5384615384615384 0.25 sunflower 0.88 0.7857142857142857 scorpion 0.4 0.4166666666666667 revolver 0.88 0.88 laptop 0.6666666666666666 0.6666666666666666 ibis 0.0 nan
best_dense_sizes = [128, 200]
It looks like the best sizes for dense layers are [128,200]. Which is a bit confusing, intuition suggest that the structure were the number of neurons in each layer is decreasing (not increasing) would work better.
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
dropout = [0.2]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_activations = ["sigmoid"]
denses_sizeses = [[100], [200], [300], [400], [500], [600], [700], [1000]]
models = []
names = []
for dense_sizes in denses_sizeses:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{str(dense_sizes)}")
compare_models(models=models, names=names)
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: [100] Number of params: 227302 Model name: [200] Number of params: 434802 Model name: [300] Number of params: 642302 Model name: [400] Number of params: 849802 Model name: [500] Number of params: 1057302 Model name: [600] Number of params: 1264802 Model name: [700] Number of params: 1472302 Model name: [1000] Number of params: 2094802 Traning times: [100]: 24.011345624923706 [200]: 27.216259002685547 [300]: 21.5516037940979 [400]: 24.166484832763672 [500]: 13.766542434692383 [600]: 16.935436725616455 [700]: 24.389729261398315 [1000]: 15.999080181121826 Best val accuracies: [100]: 0.7247437834739685 [200]: 0.7833089232444763 [300]: 0.765739381313324 [400]: 0.7920936942100525 [500]: 0.7818447947502136 [600]: 0.8023425936698914 [700]: 0.8023425936698914 [1000]: 0.6559297442436218
C:\Users\Janek\AppData\Local\Temp/ipykernel_7768/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
[100] Label Recall Precision Worst recall for: watch 0.0 Worst precision for: watch 0.0 airplanes 0.8888888888888888 0.75 Motorbikes 1.0 0.9642857142857143 Faces 0.7407407407407407 0.9090909090909091 Faces_easy 0.9629629629629629 1.0 watch 0.0 nan Leopards 0.9259259259259259 0.8928571428571429 bonsai 0.5555555555555556 0.6521739130434783 car_side 1.0 0.8181818181818182 ketch 0.7407407407407407 0.6896551724137931 chandelier 0.5925925925925926 0.6153846153846154 hawksbill 0.8148148148148148 0.6285714285714286 grand_piano 0.7777777777777778 0.9545454545454546 brain 0.6666666666666666 0.8181818181818182 butterfly 0.7037037037037037 0.7037037037037037 helicopter 0.5384615384615384 0.4827586206896552 menorah 0.8846153846153846 1.0 kangaroo 0.6153846153846154 0.48484848484848486 starfish 0.6923076923076923 0.23376623376623376 trilobite 0.9230769230769231 0.96 buddha 0.38461538461538464 0.5882352941176471 ewer 0.6538461538461539 0.7391304347826086 sunflower 0.8 0.9523809523809523 scorpion 0.36 0.6923076923076923 revolver 0.84 0.875 laptop 0.6666666666666666 0.8 ibis 0.5416666666666666 0.52 [200] Label Recall Precision Worst recall for: scorpion 0.48 Worst precision for: starfish 0.7307692307692307 airplanes 0.8148148148148148 0.9565217391304348 Motorbikes 1.0 0.9310344827586207 Faces 0.7037037037037037 0.95 Faces_easy 0.9629629629629629 0.9629629629629629 watch 0.6296296296296297 0.8095238095238095 Leopards 1.0 0.9310344827586207 bonsai 0.5925925925925926 0.8888888888888888 car_side 0.9629629629629629 0.9285714285714286 ketch 0.7407407407407407 0.8695652173913043 chandelier 0.8148148148148148 0.7096774193548387 hawksbill 0.7407407407407407 0.6896551724137931 grand_piano 0.7777777777777778 1.0 brain 0.8148148148148148 0.88 butterfly 0.5555555555555556 0.6521739130434783 helicopter 0.6923076923076923 0.5454545454545454 menorah 0.9615384615384616 0.8928571428571429 kangaroo 0.7692307692307693 0.625 starfish 0.7307692307692307 0.4318181818181818 trilobite 1.0 0.9629629629629629 buddha 0.6538461538461539 0.68 ewer 0.7307692307692307 0.6129032258064516 sunflower 0.84 0.9545454545454546 scorpion 0.48 0.6666666666666666 revolver 0.88 0.88 laptop 0.7083333333333334 0.7391304347826086 ibis 0.6666666666666666 0.5714285714285714 [300] Label Recall Precision Worst recall for: bonsai 0.5185185185185185 Worst precision for: ibis 0.5416666666666666 airplanes 0.7777777777777778 0.9545454545454546 Motorbikes 1.0 0.9642857142857143 Faces 0.5555555555555556 0.9375 Faces_easy 0.9629629629629629 0.8666666666666667 watch 0.6296296296296297 0.7727272727272727 Leopards 1.0 0.9310344827586207 bonsai 0.5185185185185185 0.7777777777777778 car_side 1.0 0.8181818181818182 ketch 0.7777777777777778 0.7777777777777778 chandelier 0.7407407407407407 0.6451612903225806 hawksbill 0.7777777777777778 0.6 grand_piano 0.6666666666666666 0.9473684210526315 brain 0.7407407407407407 0.9523809523809523 butterfly 0.7037037037037037 0.6785714285714286 helicopter 0.5384615384615384 0.5833333333333334 menorah 0.8846153846153846 0.8518518518518519 kangaroo 0.8076923076923077 0.5833333333333334 starfish 0.5769230769230769 0.4838709677419355 trilobite 1.0 0.896551724137931 buddha 0.5384615384615384 0.7777777777777778 ewer 0.8461538461538461 0.6470588235294118 sunflower 0.76 0.95 scorpion 0.6 0.4838709677419355 revolver 0.92 0.8846153846153846 laptop 0.5833333333333334 0.7368421052631579 ibis 0.5416666666666666 0.4482758620689655 [400] Label Recall Precision Worst recall for: helicopter 0.5 Worst precision for: starfish 0.7307692307692307 airplanes 0.8888888888888888 0.8888888888888888 Motorbikes 1.0 1.0 Faces 0.7777777777777778 1.0 Faces_easy 0.9629629629629629 0.9285714285714286 watch 0.8148148148148148 0.9166666666666666 Leopards 1.0 0.9310344827586207 bonsai 0.5185185185185185 0.7777777777777778 car_side 1.0 0.8709677419354839 ketch 0.8148148148148148 0.8461538461538461 chandelier 0.7777777777777778 0.75 hawksbill 0.8518518518518519 0.575 grand_piano 0.7407407407407407 1.0 brain 0.7037037037037037 0.8260869565217391 butterfly 0.6296296296296297 0.85 helicopter 0.5 0.6190476190476191 menorah 0.8461538461538461 0.7586206896551724 kangaroo 0.8076923076923077 0.5121951219512195 starfish 0.7307692307692307 0.48717948717948717 trilobite 1.0 1.0 buddha 0.5384615384615384 0.8235294117647058 ewer 0.6923076923076923 0.6428571428571429 sunflower 0.88 0.9565217391304348 scorpion 0.6 0.6 revolver 0.88 0.8148148148148148 laptop 0.75 0.6923076923076923 ibis 0.5833333333333334 0.7368421052631579 [500] Label Recall Precision Worst recall for: scorpion 0.36 Worst precision for: starfish 0.7307692307692307 airplanes 0.9259259259259259 0.9615384615384616 Motorbikes 0.9629629629629629 1.0 Faces 0.8888888888888888 1.0 Faces_easy 0.9629629629629629 0.9285714285714286 watch 0.7407407407407407 0.7692307692307693 Leopards 1.0 0.9 bonsai 0.5555555555555556 0.75 car_side 0.9629629629629629 0.896551724137931 ketch 0.8148148148148148 0.9166666666666666 chandelier 0.8148148148148148 0.7857142857142857 hawksbill 0.7037037037037037 0.6785714285714286 grand_piano 0.7037037037037037 0.95 brain 0.8148148148148148 0.7333333333333333 butterfly 0.48148148148148145 0.6842105263157895 helicopter 0.5384615384615384 0.7368421052631579 menorah 0.9615384615384616 0.9259259259259259 kangaroo 0.8846153846153846 0.5111111111111111 starfish 0.7307692307692307 0.40425531914893614 trilobite 1.0 0.9629629629629629 buddha 0.5 0.7647058823529411 ewer 0.8076923076923077 0.5526315789473685 sunflower 0.76 1.0 scorpion 0.36 0.47368421052631576 revolver 0.88 0.88 laptop 0.5 0.7058823529411765 ibis 0.5416666666666666 0.52 [600] Label Recall Precision Worst recall for: scorpion 0.44 Worst precision for: ewer 0.8076923076923077 airplanes 0.9259259259259259 0.8928571428571429 Motorbikes 1.0 0.9642857142857143 Faces 0.8518518518518519 0.8518518518518519 Faces_easy 0.9629629629629629 0.9629629629629629 watch 0.7407407407407407 0.8695652173913043 Leopards 1.0 0.9310344827586207 bonsai 0.5925925925925926 0.7272727272727273 car_side 1.0 0.9642857142857143 ketch 0.8148148148148148 0.7857142857142857 chandelier 0.7777777777777778 0.7241379310344828 hawksbill 0.6666666666666666 0.6923076923076923 grand_piano 0.8148148148148148 1.0 brain 0.7037037037037037 0.8636363636363636 butterfly 0.7037037037037037 0.9047619047619048 helicopter 0.7307692307692307 0.5757575757575758 menorah 0.8846153846153846 0.9583333333333334 kangaroo 0.7307692307692307 0.5135135135135135 starfish 0.7692307692307693 0.5714285714285714 trilobite 0.9230769230769231 0.9230769230769231 buddha 0.5384615384615384 0.875 ewer 0.8076923076923077 0.5121951219512195 sunflower 0.88 1.0 scorpion 0.44 0.8461538461538461 revolver 0.84 0.9545454545454546 laptop 0.7083333333333334 0.7083333333333334 ibis 0.7083333333333334 0.5666666666666667 [700] Label Recall Precision Worst recall for: scorpion 0.4 Worst precision for: ibis 0.6666666666666666 airplanes 0.9259259259259259 0.9259259259259259 Motorbikes 1.0 0.9642857142857143 Faces 0.7407407407407407 0.9523809523809523 Faces_easy 0.9629629629629629 0.9285714285714286 watch 0.7407407407407407 0.8 Leopards 1.0 0.9310344827586207 bonsai 0.5925925925925926 0.8 car_side 1.0 0.9 ketch 0.8518518518518519 0.92 chandelier 0.8148148148148148 0.6285714285714286 hawksbill 0.8148148148148148 0.7586206896551724 grand_piano 0.7037037037037037 0.95 brain 0.7407407407407407 0.8333333333333334 butterfly 0.6296296296296297 0.85 helicopter 0.7692307692307693 0.625 menorah 0.9230769230769231 0.9230769230769231 kangaroo 0.8461538461538461 0.6285714285714286 starfish 0.6923076923076923 0.5806451612903226 trilobite 1.0 0.896551724137931 buddha 0.5 0.7647058823529411 ewer 0.8461538461538461 0.55 sunflower 0.8 0.9523809523809523 scorpion 0.4 0.5882352941176471 revolver 0.88 0.9565217391304348 laptop 0.6666666666666666 0.7619047619047619 ibis 0.6666666666666666 0.5333333333333333 [1000] Label Recall Precision Worst recall for: Faces_easy 0.0 Worst precision for: Faces_easy 0.0 airplanes 0.8888888888888888 0.7272727272727273 Motorbikes 0.9629629629629629 0.9285714285714286 Faces 0.8518518518518519 0.5609756097560976 Faces_easy 0.0 nan watch 0.5925925925925926 0.6666666666666666 Leopards 1.0 0.84375 bonsai 0.6296296296296297 0.7727272727272727 car_side 0.9629629629629629 0.896551724137931 ketch 0.0 nan chandelier 0.7777777777777778 0.44680851063829785 hawksbill 0.7777777777777778 0.4375 grand_piano 0.6296296296296297 0.8947368421052632 brain 0.0 nan butterfly 0.6666666666666666 0.6666666666666666 helicopter 0.4230769230769231 0.55 menorah 0.9230769230769231 0.7741935483870968 kangaroo 0.7692307692307693 0.47619047619047616 starfish 0.6153846153846154 0.4 trilobite 1.0 0.8666666666666667 buddha 0.5 0.6190476190476191 ewer 0.6538461538461539 0.6296296296296297 sunflower 0.8 1.0 scorpion 0.4 0.4 revolver 0.84 0.75 laptop 0.6666666666666666 0.5925925925925926 ibis 0.5 0.5454545454545454
Here we can see that one dense layer seems to work a bit better than 2 layer structure. We performed this test a couple of times more, to test if it wasn't by a coincidence. It seems it wasn't. One layer seems to be better for this specific problem taking into account our structure of the model. Also, this version of the dense layer structure less times struggles with problem described above (not being able to recognize one class at all). Further more, confussion matrices look more smoother.
Below, we wanted to test also, what would happen if we added more layers?
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
dropout = [0.2]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_activations = ["sigmoid", "sigmoid", "relu"]
denses_sizeses = [[100, 100, 100], [200, 200, 100], [300, 200, 100], [400, 200, 200], [300, 300, 100], [300, 300, 300]]
models = []
names = []
for dense_sizes in denses_sizeses:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{str(dense_sizes)}")
compare_models(models=models, names=names)
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: [100, 100, 100] Number of params: 247502 Model name: [200, 200, 100] Number of params: 492502 Model name: [300, 200, 100] Number of params: 717402 Model name: [400, 200, 200] Number of params: 965002 Model name: [300, 300, 100] Number of params: 757502 Model name: [300, 300, 300] Number of params: 822902 Traning times: [100, 100, 100]: 13.834101438522339 [200, 200, 100]: 15.286413192749023 [300, 200, 100]: 21.284404516220093 [400, 200, 200]: 18.216601610183716 [300, 300, 100]: 21.422527551651 [300, 300, 300]: 32.649210691452026 Best val accuracies: [100, 100, 100]: 0.5490483045578003 [200, 200, 100]: 0.5988286733627319 [300, 200, 100]: 0.5900439023971558 [400, 200, 200]: 0.6559297442436218 [300, 300, 100]: 0.5754026174545288 [300, 300, 300]: 0.6105417013168335
C:\Users\Janek\AppData\Local\Temp/ipykernel_7768/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
[100, 100, 100] Label Recall Precision Worst recall for: brain 0.0 Worst precision for: brain 0.0 airplanes 0.9629629629629629 0.7428571428571429 Motorbikes 0.8518518518518519 1.0 Faces 0.7407407407407407 0.8695652173913043 Faces_easy 0.9629629629629629 0.9629629629629629 watch 0.7777777777777778 0.40384615384615385 Leopards 0.9629629629629629 0.8387096774193549 bonsai 0.48148148148148145 0.22413793103448276 car_side 0.9259259259259259 0.6944444444444444 ketch 1.0 0.5869565217391305 chandelier 0.8888888888888888 0.4444444444444444 hawksbill 0.8148148148148148 0.36065573770491804 grand_piano 0.6296296296296297 1.0 brain 0.0 nan butterfly 0.18518518518518517 0.3125 helicopter 0.0 nan menorah 0.0 nan kangaroo 0.8461538461538461 0.2972972972972973 starfish 0.0 nan trilobite 0.9615384615384616 0.8620689655172413 buddha 0.0 nan ewer 0.0 nan sunflower 0.64 0.8421052631578947 scorpion 0.0 nan revolver 0.6 0.8823529411764706 laptop 0.0 nan ibis 0.7916666666666666 0.2923076923076923 [200, 200, 100] Label Recall Precision Worst recall for: watch 0.0 Worst precision for: watch 0.0 airplanes 0.9629629629629629 0.4406779661016949 Motorbikes 0.8518518518518519 0.92 Faces 0.7777777777777778 0.9545454545454546 Faces_easy 1.0 0.9 watch 0.0 nan Leopards 0.8148148148148148 0.9565217391304348 bonsai 0.5555555555555556 0.6818181818181818 car_side 1.0 0.9642857142857143 ketch 0.9629629629629629 0.6341463414634146 chandelier 0.8888888888888888 0.5333333333333333 hawksbill 0.7407407407407407 0.5882352941176471 grand_piano 0.7777777777777778 0.84 brain 0.5925925925925926 0.42105263157894735 butterfly 0.0 nan helicopter 0.0 nan menorah 0.0 nan kangaroo 0.8461538461538461 0.4230769230769231 starfish 0.7692307692307693 0.18867924528301888 trilobite 1.0 0.7647058823529411 buddha 0.0 nan ewer 0.0 nan sunflower 0.84 0.9545454545454546 scorpion 0.0 nan revolver 0.92 0.6388888888888888 laptop 0.8333333333333334 0.4878048780487805 ibis 0.0 nan [300, 200, 100] Label Recall Precision Worst recall for: airplanes 0.0 Worst precision for: airplanes 0.0 airplanes 0.0 nan Motorbikes 0.9259259259259259 0.9615384615384616 Faces 0.8888888888888888 0.75 Faces_easy 1.0 0.9 watch 0.0 nan Leopards 0.0 nan bonsai 0.5925925925925926 0.3076923076923077 car_side 0.9259259259259259 0.9259259259259259 ketch 0.8888888888888888 0.7741935483870968 chandelier 0.7407407407407407 0.5 hawksbill 0.6666666666666666 0.5294117647058824 grand_piano 0.7777777777777778 0.8076923076923077 brain 0.6666666666666666 0.5 butterfly 0.6296296296296297 0.3541666666666667 helicopter 0.4230769230769231 0.44 menorah 0.7307692307692307 0.7037037037037037 kangaroo 0.6923076923076923 0.46153846153846156 starfish 0.0 nan trilobite 1.0 0.896551724137931 buddha 0.6538461538461539 0.3953488372093023 ewer 0.0 nan sunflower 0.76 0.8636363636363636 scorpion 0.0 nan revolver 0.92 0.575 laptop 0.6666666666666666 0.6153846153846154 ibis 0.7916666666666666 0.38 [400, 200, 200] Label Recall Precision Worst recall for: brain 0.0 Worst precision for: brain 0.0 airplanes 0.9259259259259259 0.9259259259259259 Motorbikes 0.8888888888888888 0.9230769230769231 Faces 0.9629629629629629 0.6842105263157895 Faces_easy 1.0 0.9 watch 0.7777777777777778 0.6363636363636364 Leopards 0.9259259259259259 0.8928571428571429 bonsai 0.48148148148148145 0.5416666666666666 car_side 1.0 0.7941176470588235 ketch 0.9259259259259259 0.7352941176470589 chandelier 0.8888888888888888 0.43636363636363634 hawksbill 0.7037037037037037 0.6551724137931034 grand_piano 0.8148148148148148 0.7586206896551724 brain 0.0 nan butterfly 0.0 nan helicopter 0.4230769230769231 0.55 menorah 0.7692307692307693 0.7407407407407407 kangaroo 0.0 nan starfish 0.6923076923076923 0.2903225806451613 trilobite 1.0 0.7878787878787878 buddha 0.0 nan ewer 0.7692307692307693 0.6060606060606061 sunflower 0.92 0.6388888888888888 scorpion 0.6 0.36585365853658536 revolver 0.72 0.9 laptop 0.7083333333333334 0.7083333333333334 ibis 0.0 nan [300, 300, 100] Label Recall Precision Worst recall for: Faces_easy 0.0 Worst precision for: Faces_easy 0.0 airplanes 0.8148148148148148 0.9565217391304348 Motorbikes 0.9629629629629629 0.7878787878787878 Faces 0.8518518518518519 0.6571428571428571 Faces_easy 0.0 nan watch 0.0 nan Leopards 0.8518518518518519 0.92 bonsai 0.4444444444444444 0.75 car_side 0.9629629629629629 0.9285714285714286 ketch 0.8888888888888888 0.6486486486486487 chandelier 0.8518518518518519 0.575 hawksbill 0.7777777777777778 0.525 grand_piano 0.8148148148148148 0.6470588235294118 brain 0.7777777777777778 0.4772727272727273 butterfly 0.6666666666666666 0.35294117647058826 helicopter 0.6153846153846154 0.3404255319148936 menorah 0.7692307692307693 0.5555555555555556 kangaroo 0.5769230769230769 0.42857142857142855 starfish 0.0 nan trilobite 0.0 nan buddha 0.0 nan ewer 0.0 nan sunflower 0.6 0.8823529411764706 scorpion 0.0 nan revolver 0.76 0.9047619047619048 laptop 0.5 0.6666666666666666 ibis 0.7083333333333334 0.1650485436893204 [300, 300, 300] Label Recall Precision Worst recall for: Leopards 0.0 Worst precision for: Leopards 0.0 airplanes 0.9629629629629629 0.9285714285714286 Motorbikes 0.9259259259259259 0.9259259259259259 Faces 0.8888888888888888 0.9230769230769231 Faces_easy 1.0 0.9310344827586207 watch 0.8148148148148148 0.6470588235294118 Leopards 0.0 nan bonsai 0.5925925925925926 0.4 car_side 1.0 0.7941176470588235 ketch 0.9629629629629629 0.65 chandelier 0.7407407407407407 0.35714285714285715 hawksbill 0.0 nan grand_piano 0.8888888888888888 0.5581395348837209 brain 0.0 nan butterfly 0.5555555555555556 0.375 helicopter 0.46153846153846156 0.3 menorah 0.8076923076923077 0.9130434782608695 kangaroo 0.8076923076923077 0.38181818181818183 starfish 0.0 nan trilobite 0.9615384615384616 0.6756756756756757 buddha 0.0 nan ewer 0.6923076923076923 0.3673469387755102 sunflower 0.72 0.6666666666666666 scorpion 0.0 nan revolver 0.88 0.8148148148148148 laptop 0.75 0.6428571428571429 ibis 0.0 nan
Tenedency can be observed, the more layers in the dense structure, the worse performance of the model is. We suspect that this is the case, because of the small amount of classes in the dataset (Caltech 101 is a small datasets, and we prune it further to 26 classes).
Here we try to find the best dropout value.
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_sizes = best_dense_sizes
dense_activations = ["sigmoid", "relu"]
dropouts = [[0]*3,[0.1]*3, [0.2]*3,[0.3]*3 ,[0.4]*3, [0.5]*3, [0.7]*3]
models = []
names = []
for dropout in dropouts:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{str(dropout[0])}")
compare_models(models=models, names=names)
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: 0 Number of params: 313074 Model name: 0.1 Number of params: 313074 Model name: 0.2 Number of params: 313074 Model name: 0.3 Number of params: 313074 Model name: 0.4 Number of params: 313074 Model name: 0.5 Number of params: 313074 Model name: 0.7 Number of params: 313074 Traning times: 0: 21.006017446517944 0.1: 17.807413816452026 0.2: 27.036015033721924 0.3: 24.6343891620636 0.4: 17.742249011993408 0.5: 32.844088554382324 0.7: 19.605994939804077 Best val accuracies: 0: 0.8111273646354675 0.1: 0.7686676383018494 0.2: 0.803806722164154 0.3: 0.6866764426231384 0.4: 0.6134699583053589 0.5: 0.6486091017723083 0.7: 0.4245973527431488
/tmp/ipykernel_113926/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
0 Label Recall Precision Worst recall for: scorpion 0.0 Worst precision for: scorpion 0.0 airplanes 0.9629629629629629 0.9285714285714286 Motorbikes 0.9629629629629629 0.9629629629629629 Faces_easy 1.0 0.9642857142857143 Faces 0.8518518518518519 0.92 watch 0.7407407407407407 0.8695652173913043 Leopards 1.0 0.9 bonsai 0.6296296296296297 0.9444444444444444 car_side 0.9629629629629629 0.8666666666666667 ketch 1.0 0.8709677419354839 chandelier 0.8148148148148148 0.6875 hawksbill 0.7037037037037037 0.8260869565217391 grand_piano 0.9629629629629629 0.896551724137931 brain 0.8518518518518519 0.5 butterfly 0.6296296296296297 0.68 helicopter 0.7307692307692307 0.6785714285714286 menorah 0.8076923076923077 0.7777777777777778 starfish 0.7307692307692307 0.59375 trilobite 0.9615384615384616 0.9615384615384616 kangaroo 0.8461538461538461 0.7586206896551724 ewer 0.8076923076923077 0.7 buddha 0.7307692307692307 0.9047619047619048 sunflower 0.76 0.9047619047619048 scorpion 0.0 nan revolver 0.8 0.9090909090909091 laptop 0.75 0.75 ibis 0.7916666666666666 0.6785714285714286 0.1 Label Recall Precision Worst recall for: menorah 0.0 Worst precision for: menorah 0.0 airplanes 0.9629629629629629 0.8125 Motorbikes 0.9259259259259259 0.9615384615384616 Faces_easy 1.0 0.9310344827586207 Faces 0.8518518518518519 0.9583333333333334 watch 0.8518518518518519 0.5897435897435898 Leopards 0.9629629629629629 0.9285714285714286 bonsai 0.7037037037037037 0.76 car_side 1.0 0.8709677419354839 ketch 0.8888888888888888 0.8 chandelier 0.6666666666666666 0.75 hawksbill 0.6666666666666666 0.782608695652174 grand_piano 0.9629629629629629 0.9629629629629629 brain 0.7407407407407407 0.6896551724137931 butterfly 0.48148148148148145 0.7647058823529411 helicopter 0.8076923076923077 0.9130434782608695 menorah 0.0 nan starfish 0.6923076923076923 0.5625 trilobite 0.9615384615384616 0.9259259259259259 kangaroo 0.8076923076923077 0.6363636363636364 ewer 0.7307692307692307 0.6551724137931034 buddha 0.7307692307692307 0.7916666666666666 sunflower 0.64 0.8 scorpion 0.4 0.2702702702702703 revolver 0.84 0.9130434782608695 laptop 0.7916666666666666 0.9047619047619048 ibis 0.75 0.6 0.2 Label Recall Precision Worst recall for: butterfly 0.48148148148148145 Worst precision for: ewer 0.5769230769230769 airplanes 0.9629629629629629 1.0 Motorbikes 0.9629629629629629 0.9629629629629629 Faces_easy 1.0 1.0 Faces 0.9259259259259259 0.9259259259259259 watch 0.8148148148148148 0.8461538461538461 Leopards 0.9629629629629629 0.9629629629629629 bonsai 0.5925925925925926 0.8888888888888888 car_side 1.0 0.9310344827586207 ketch 0.9629629629629629 0.896551724137931 chandelier 0.7777777777777778 0.7777777777777778 hawksbill 0.7037037037037037 0.5 grand_piano 0.8888888888888888 1.0 brain 0.7037037037037037 0.76 butterfly 0.48148148148148145 0.7222222222222222 helicopter 0.8076923076923077 0.84 menorah 0.8076923076923077 0.7777777777777778 starfish 0.6153846153846154 0.5925925925925926 trilobite 0.9230769230769231 0.9230769230769231 kangaroo 0.7307692307692307 0.5757575757575758 ewer 0.5769230769230769 0.4838709677419355 buddha 0.6923076923076923 0.9 sunflower 0.72 1.0 scorpion 0.6 0.7142857142857143 revolver 0.96 0.8571428571428571 laptop 0.875 0.875 ibis 0.75 0.5142857142857142 0.3 Label Recall Precision Worst recall for: Faces_easy 0.0 Worst precision for: Faces_easy 0.0 airplanes 0.8888888888888888 0.8888888888888888 Motorbikes 0.9259259259259259 1.0 Faces_easy 0.0 nan Faces 0.7777777777777778 0.9130434782608695 watch 0.0 nan Leopards 0.8518518518518519 1.0 bonsai 0.5555555555555556 0.8333333333333334 car_side 0.8888888888888888 0.8888888888888888 ketch 0.8518518518518519 0.8214285714285714 chandelier 0.7037037037037037 0.5135135135135135 hawksbill 0.7037037037037037 0.6551724137931034 grand_piano 0.8518518518518519 0.92 brain 0.7037037037037037 0.5428571428571428 butterfly 0.5555555555555556 0.8333333333333334 helicopter 0.0 nan menorah 0.7307692307692307 0.7037037037037037 starfish 0.46153846153846156 0.631578947368421 trilobite 0.9615384615384616 0.9259259259259259 kangaroo 0.6538461538461539 0.7391304347826086 ewer 0.5 0.3939393939393939 buddha 0.7307692307692307 0.5277777777777778 sunflower 0.56 0.9333333333333333 scorpion 0.64 0.3404255319148936 revolver 0.84 0.875 laptop 0.75 0.8571428571428571 ibis 0.9166666666666666 0.22916666666666666 0.4 Label Recall Precision Worst recall for: brain 0.0 Worst precision for: brain 0.0 airplanes 0.5555555555555556 0.7894736842105263 Motorbikes 0.8888888888888888 1.0 Faces_easy 0.7407407407407407 1.0 Faces 0.5925925925925926 0.7619047619047619 watch 0.8518518518518519 0.359375 Leopards 0.2962962962962963 1.0 bonsai 0.4074074074074074 1.0 car_side 0.9629629629629629 0.8666666666666667 ketch 0.8148148148148148 0.88 chandelier 0.5555555555555556 0.625 hawksbill 0.5185185185185185 0.5 grand_piano 0.6666666666666666 1.0 brain 0.0 nan butterfly 0.37037037037037035 0.5263157894736842 helicopter 0.6153846153846154 0.5925925925925926 menorah 0.6153846153846154 0.8 starfish 0.5769230769230769 0.2112676056338028 trilobite 0.46153846153846156 1.0 kangaroo 0.5769230769230769 0.46875 ewer 0.6923076923076923 0.375 buddha 0.34615384615384615 0.8181818181818182 sunflower 0.6 1.0 scorpion 0.0 nan revolver 0.52 1.0 laptop 0.4583333333333333 0.7857142857142857 ibis 0.8333333333333334 0.1834862385321101 0.5 Label Recall Precision Worst recall for: ibis 0.0 Worst precision for: ibis 0.0 airplanes 0.9259259259259259 0.6944444444444444 Motorbikes 0.8888888888888888 1.0 Faces_easy 1.0 0.75 Faces 0.48148148148148145 1.0 watch 0.5925925925925926 0.5333333333333333 Leopards 0.7037037037037037 0.9047619047619048 bonsai 0.4074074074074074 0.6470588235294118 car_side 0.9629629629629629 0.5652173913043478 ketch 0.8888888888888888 0.8888888888888888 chandelier 0.4444444444444444 0.631578947368421 hawksbill 0.7777777777777778 0.35 grand_piano 0.7407407407407407 0.9090909090909091 brain 0.5925925925925926 0.9411764705882353 butterfly 0.25925925925925924 0.7 helicopter 0.46153846153846156 0.6666666666666666 menorah 0.4230769230769231 0.9166666666666666 starfish 0.5 0.24074074074074073 trilobite 0.2692307692307692 1.0 kangaroo 0.7692307692307693 0.25 ewer 0.8846153846153846 0.5897435897435898 buddha 0.5 0.4642857142857143 sunflower 0.4 1.0 scorpion 0.24 0.5454545454545454 revolver 0.6 1.0 laptop 0.8333333333333334 0.6451612903225806 ibis 0.0 nan 0.7 Label Recall Precision Worst recall for: watch 0.0 Worst precision for: watch 0.0 airplanes 0.6296296296296297 0.6538461538461539 Motorbikes 0.7037037037037037 0.95 Faces_easy 1.0 0.9310344827586207 Faces 0.7407407407407407 0.7407407407407407 watch 0.0 nan Leopards 0.7777777777777778 0.7241379310344828 bonsai 0.14814814814814814 1.0 car_side 0.5925925925925926 0.6956521739130435 ketch 0.9629629629629629 0.4 chandelier 0.14814814814814814 0.4444444444444444 hawksbill 0.5555555555555556 0.40540540540540543 grand_piano 0.5555555555555556 0.8823529411764706 brain 0.14814814814814814 0.3333333333333333 butterfly 0.0 nan helicopter 0.46153846153846156 0.48 menorah 0.2692307692307692 1.0 starfish 0.0 nan trilobite 0.0 nan kangaroo 0.8076923076923077 0.10824742268041238 ewer 0.0 nan buddha 0.0 nan sunflower 0.2 1.0 scorpion 0.0 nan revolver 0.76 0.6333333333333333 laptop 0.7083333333333334 0.5862068965517241 ibis 0.5416666666666666 0.1368421052631579
Results are very confusing. Apparently best droupout seems to be no dropout at all.. However 0.2 seems to be pretty close so we will examine few more examples near 0 and 0.2. On average, when test previously, dropout around this rate seemed the best. But this result is very interesting, as every network was trained on the same data. Also the first model, as said before for some reason wasn't able to recognize one class, this time it was scorpion.
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
dropout = [0.2]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_sizes = best_dense_sizes
dense_activations = ["sigmoid", "relu"]
dropouts = [[0.01]*3, [0.02]*3,[0.18]*3 ,[0.22]*3, [0.23]*3, [0.19]*3]
models = []
names = []
for dropout in dropouts:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{str(dropout[0])}")
compare_models(models=models, names=names)
#Droput 0.1 seems best but bulsshit
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: 0.01 Number of params: 313074 Model name: 0.02 Number of params: 313074 Model name: 0.18 Number of params: 313074 Model name: 0.22 Number of params: 313074 Model name: 0.23 Number of params: 313074 Model name: 0.19 Number of params: 313074 Traning times: 0.01: 16.013524770736694 0.02: 20.15576148033142 0.18: 24.827941179275513 0.22: 33.22116708755493 0.23: 31.719167947769165 0.19: 30.093747854232788 Best val accuracies: 0.01: 0.7789165377616882 0.02: 0.7540263533592224 0.18: 0.7628111243247986 0.22: 0.7964860796928406 0.23: 0.7833089232444763 0.19: 0.7730600237846375
0.01 Label Recall Precision Worst recall for: scorpion 0.44 Worst precision for: scorpion 0.44 airplanes 0.8888888888888888 0.96 Motorbikes 0.9259259259259259 0.9259259259259259 Faces_easy 0.9259259259259259 0.9259259259259259 Faces 1.0 0.9 watch 0.8888888888888888 0.8 Leopards 1.0 0.9642857142857143 bonsai 0.6296296296296297 0.8095238095238095 car_side 0.9629629629629629 0.896551724137931 ketch 0.8518518518518519 0.8518518518518519 chandelier 0.8148148148148148 0.7333333333333333 hawksbill 0.7407407407407407 0.8333333333333334 grand_piano 0.9259259259259259 0.8064516129032258 brain 0.8888888888888888 0.5853658536585366 butterfly 0.5185185185185185 0.6363636363636364 helicopter 0.5384615384615384 0.6666666666666666 menorah 0.7692307692307693 0.9090909090909091 starfish 0.6538461538461539 0.6538461538461539 trilobite 0.9615384615384616 0.9259259259259259 kangaroo 0.5769230769230769 0.5769230769230769 ewer 0.5 0.7222222222222222 buddha 0.6153846153846154 0.8 sunflower 0.8 0.7692307692307693 scorpion 0.44 0.44 revolver 0.84 0.8076923076923077 laptop 0.5833333333333334 0.8235294117647058 ibis 0.75 0.4864864864864865 0.02 Label Recall Precision Worst recall for: helicopter 0.0 Worst precision for: helicopter 0.0 airplanes 0.9629629629629629 0.9285714285714286 Motorbikes 0.9629629629629629 0.9629629629629629 Faces_easy 0.9259259259259259 0.9259259259259259 Faces 0.8888888888888888 0.8571428571428571 watch 0.8148148148148148 0.6666666666666666 Leopards 0.7037037037037037 0.95 bonsai 0.5925925925925926 1.0 car_side 1.0 0.9 ketch 0.8518518518518519 0.8846153846153846 chandelier 0.7777777777777778 0.875 hawksbill 0.7777777777777778 0.7 grand_piano 0.8518518518518519 0.8214285714285714 brain 0.8148148148148148 0.88 butterfly 0.7777777777777778 0.5833333333333334 helicopter 0.0 nan menorah 0.7692307692307693 0.7142857142857143 starfish 0.6923076923076923 0.5142857142857142 trilobite 0.9615384615384616 1.0 kangaroo 0.6153846153846154 0.5925925925925926 ewer 0.7692307692307693 0.37735849056603776 buddha 0.5 0.52 sunflower 0.68 0.9444444444444444 scorpion 0.52 0.65 revolver 0.8 0.8333333333333334 laptop 0.5833333333333334 0.7777777777777778 ibis 0.5833333333333334 0.4375
/tmp/ipykernel_113926/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
0.18 Label Recall Precision Worst recall for: chandelier 0.0 Worst precision for: chandelier 0.0 airplanes 0.9629629629629629 0.9629629629629629 Motorbikes 1.0 0.9310344827586207 Faces_easy 0.9629629629629629 0.9629629629629629 Faces 0.8888888888888888 0.8275862068965517 watch 0.8888888888888888 0.6153846153846154 Leopards 0.9629629629629629 0.896551724137931 bonsai 0.5555555555555556 0.9375 car_side 0.9629629629629629 1.0 ketch 0.8518518518518519 0.575 chandelier 0.0 nan hawksbill 0.6666666666666666 0.72 grand_piano 0.8148148148148148 1.0 brain 0.7777777777777778 0.875 butterfly 0.7037037037037037 0.5428571428571428 helicopter 0.6153846153846154 0.64 menorah 0.8076923076923077 0.6774193548387096 starfish 0.7692307692307693 0.6666666666666666 trilobite 1.0 0.9285714285714286 kangaroo 0.6923076923076923 0.782608695652174 ewer 0.6153846153846154 0.7619047619047619 buddha 0.5769230769230769 0.7142857142857143 sunflower 0.76 0.8636363636363636 scorpion 0.68 0.3617021276595745 revolver 0.8 0.9090909090909091 laptop 0.5416666666666666 0.9285714285714286 ibis 0.6666666666666666 0.5161290322580645 0.22 Label Recall Precision Worst recall for: butterfly 0.37037037037037035 Worst precision for: starfish 0.5769230769230769 airplanes 0.9259259259259259 1.0 Motorbikes 0.9629629629629629 0.9285714285714286 Faces_easy 0.9259259259259259 0.8620689655172413 Faces 0.8518518518518519 1.0 watch 0.8148148148148148 0.9166666666666666 Leopards 0.9629629629629629 0.9629629629629629 bonsai 0.48148148148148145 1.0 car_side 0.9629629629629629 0.8387096774193549 ketch 0.8888888888888888 0.9230769230769231 chandelier 0.8518518518518519 0.6052631578947368 hawksbill 0.5925925925925926 0.7272727272727273 grand_piano 0.8518518518518519 0.9583333333333334 brain 0.8888888888888888 0.8888888888888888 butterfly 0.37037037037037035 0.7142857142857143 helicopter 0.4230769230769231 0.7857142857142857 menorah 0.8461538461538461 0.7586206896551724 starfish 0.5769230769230769 0.36585365853658536 trilobite 0.9615384615384616 0.8928571428571429 kangaroo 0.6923076923076923 0.4864864864864865 ewer 0.8076923076923077 0.5121951219512195 buddha 0.6538461538461539 0.6538461538461539 sunflower 0.76 0.95 scorpion 0.52 0.37142857142857144 revolver 0.84 0.7777777777777778 laptop 0.75 0.8571428571428571 ibis 0.4583333333333333 0.8461538461538461 0.23 Label Recall Precision Worst recall for: helicopter 0.38461538461538464 Worst precision for: butterfly 0.5925925925925926 airplanes 0.9259259259259259 1.0 Motorbikes 0.9259259259259259 0.9259259259259259 Faces_easy 0.8888888888888888 1.0 Faces 0.9259259259259259 0.8928571428571429 watch 0.8888888888888888 0.75 Leopards 0.8888888888888888 0.8888888888888888 bonsai 0.7037037037037037 0.6333333333333333 car_side 1.0 0.84375 ketch 0.9259259259259259 0.7352941176470589 chandelier 0.7777777777777778 0.6 hawksbill 0.7407407407407407 0.6451612903225806 grand_piano 0.8148148148148148 0.9565217391304348 brain 0.8148148148148148 0.7333333333333333 butterfly 0.5925925925925926 0.3018867924528302 helicopter 0.38461538461538464 0.9090909090909091 menorah 0.8461538461538461 0.9166666666666666 starfish 0.5 0.65 trilobite 0.9230769230769231 1.0 kangaroo 0.46153846153846156 0.48 ewer 0.7307692307692307 0.6551724137931034 buddha 0.5384615384615384 0.7777777777777778 sunflower 0.64 0.9411764705882353 scorpion 0.6 0.6 revolver 0.88 0.9166666666666666 laptop 0.4583333333333333 0.8461538461538461 ibis 0.5416666666666666 0.5909090909090909 0.19 Label Recall Precision Worst recall for: scorpion 0.0 Worst precision for: scorpion 0.0 airplanes 0.9629629629629629 1.0 Motorbikes 0.9629629629629629 0.8666666666666667 Faces_easy 0.9259259259259259 0.8620689655172413 Faces 0.8518518518518519 0.8846153846153846 watch 0.9629629629629629 0.9629629629629629 Leopards 0.8148148148148148 0.9565217391304348 bonsai 0.6296296296296297 0.7727272727272727 car_side 1.0 1.0 ketch 0.9259259259259259 0.8620689655172413 chandelier 0.8518518518518519 0.6388888888888888 hawksbill 0.6666666666666666 0.782608695652174 grand_piano 0.9259259259259259 0.9615384615384616 brain 0.8148148148148148 0.7857142857142857 butterfly 0.6666666666666666 0.46153846153846156 helicopter 0.5769230769230769 0.75 menorah 0.8846153846153846 0.8214285714285714 starfish 0.6538461538461539 0.5483870967741935 trilobite 1.0 0.9629629629629629 kangaroo 0.7307692307692307 0.6551724137931034 ewer 0.7307692307692307 0.4318181818181818 buddha 0.5384615384615384 0.7368421052631579 sunflower 0.68 0.9444444444444444 scorpion 0.0 nan revolver 0.84 0.8076923076923077 laptop 0.5833333333333334 0.9333333333333333 ibis 0.7083333333333334 0.4857142857142857
Around 0 and 0.2 are still the best. Despite the fact that 0 is slightly better than 0.2 We decided to go with 0.2. Maybe it might help when testing more layers. We see that training time is a little bit higher, but learning curve is more smooth with higher dropout.
best_dropout = 0.2
This was a bit difficult to implement, but most certainly interesting. In the future, we could add here aditional preprocessing such as gaussian blur or such.
def standardize_images(files):
means = np.mean(files, axis=(0, 1, 2))
std = np.std(files, axis=(0, 1, 2))
return (files - means) / std
def normalize_images(files):
max_ = np.max(files, axis=(0, 1, 2))
min_ = np.min(files, axis=(0, 1, 2))
return files - min_/(max_ - min_)
def substract_mean(files):
mean = np.mean(files, axis=(0, 1, 2))
return files-mean
def raw_data(files):
return files.astype(np.float64)
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
activations = ["relu"]*3
dropout = [best_dropout]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_sizes = best_dense_sizes
dense_activations = ["sigmoid", "relu"]
num_of_classes = []
models = []
names = ["standardize_images", "normalize_images", "substract_mean", "raw_data"]
img_process_funcs = [standardize_images, normalize_images, substract_mean, raw_data]
for ___ in img_process_funcs:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
num_of_classes.append(26)
compare_models(models=models, names=names, img_process_funcs=img_process_funcs,
number_of_classes_to_pick=num_of_classes, same_data_for_every_process_func=True)
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: standardize_images Number of params: 313074 Model name: normalize_images Number of params: 313074 Model name: substract_mean Number of params: 313074 Model name: raw_data Number of params: 313074 Traning times: standardize_images: 25.683014631271362 normalize_images: 19.146145820617676 substract_mean: 23.219666719436646 raw_data: 21.647873163223267 Best val accuracies: standardize_images: 0.7540263533592224 normalize_images: 0.7042459845542908 substract_mean: 0.7203513979911804 raw_data: 0.6822840571403503
/tmp/ipykernel_113926/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
standardize_images Label Recall Precision Worst recall for: scorpion 0.0 Worst precision for: scorpion 0.0 airplanes 0.8888888888888888 0.96 Motorbikes 0.9259259259259259 0.9259259259259259 Faces_easy 1.0 0.9642857142857143 Faces 0.8518518518518519 0.8518518518518519 watch 0.8518518518518519 0.8846153846153846 Leopards 0.8518518518518519 0.8846153846153846 bonsai 0.5925925925925926 0.8 car_side 1.0 0.9310344827586207 ketch 0.9259259259259259 0.7142857142857143 chandelier 0.7777777777777778 0.65625 hawksbill 0.6296296296296297 0.6538461538461539 grand_piano 0.8148148148148148 0.9166666666666666 brain 0.8888888888888888 0.96 butterfly 0.48148148148148145 0.8666666666666667 helicopter 0.6153846153846154 0.5714285714285714 menorah 0.7692307692307693 0.7692307692307693 starfish 0.8461538461538461 0.41509433962264153 trilobite 0.8076923076923077 0.875 kangaroo 0.5769230769230769 0.5357142857142857 ewer 0.8846153846153846 0.45098039215686275 buddha 0.6538461538461539 0.8095238095238095 sunflower 0.8 0.9090909090909091 scorpion 0.0 nan revolver 0.84 0.7241379310344828 laptop 0.625 0.8333333333333334 ibis 0.5416666666666666 0.7222222222222222 normalize_images Label Recall Precision Worst recall for: brain 0.0 Worst precision for: brain 0.0 airplanes 0.9259259259259259 0.6410256410256411 Motorbikes 0.8888888888888888 0.9230769230769231 Faces_easy 1.0 0.9310344827586207 Faces 0.6666666666666666 0.8181818181818182 watch 0.8518518518518519 0.7931034482758621 Leopards 0.8888888888888888 0.8888888888888888 bonsai 0.8148148148148148 0.5641025641025641 car_side 0.9629629629629629 0.7878787878787878 ketch 0.7407407407407407 0.6451612903225806 chandelier 0.8518518518518519 0.5348837209302325 hawksbill 0.7037037037037037 0.5 grand_piano 0.7777777777777778 0.9545454545454546 brain 0.0 nan butterfly 0.0 nan helicopter 0.34615384615384615 0.5625 menorah 0.8846153846153846 0.7419354838709677 starfish 0.46153846153846156 0.5 trilobite 0.9615384615384616 0.6756756756756757 kangaroo 0.8846153846153846 0.359375 ewer 0.6153846153846154 0.48484848484848486 buddha 0.0 nan sunflower 0.72 1.0 scorpion 0.28 0.3684210526315789 revolver 0.68 1.0 laptop 0.625 0.7894736842105263 ibis 0.5416666666666666 0.48148148148148145 substract_mean Label Recall Precision Worst recall for: Faces 0.0 Worst precision for: Faces 0.0 airplanes 0.9259259259259259 0.8928571428571429 Motorbikes 0.9259259259259259 0.9615384615384616 Faces_easy 1.0 0.8709677419354839 Faces 0.0 nan watch 0.8888888888888888 0.8 Leopards 0.8518518518518519 0.92 bonsai 0.7037037037037037 0.6551724137931034 car_side 1.0 0.8709677419354839 ketch 0.8148148148148148 0.7333333333333333 chandelier 0.8148148148148148 0.7857142857142857 hawksbill 0.7777777777777778 0.75 grand_piano 0.0 nan brain 0.7777777777777778 0.9130434782608695 butterfly 0.48148148148148145 0.5909090909090909 helicopter 0.7307692307692307 0.7916666666666666 menorah 0.8076923076923077 0.75 starfish 0.6538461538461539 0.6071428571428571 trilobite 0.8846153846153846 0.9583333333333334 kangaroo 0.6538461538461539 0.4358974358974359 ewer 0.6923076923076923 0.5294117647058824 buddha 0.6153846153846154 0.64 sunflower 0.72 0.9473684210526315 scorpion 0.68 0.34 revolver 0.76 0.8636363636363636 laptop 0.75 0.72 ibis 0.5416666666666666 0.38235294117647056 raw_data Label Recall Precision Worst recall for: starfish 0.0 Worst precision for: starfish 0.0 airplanes 0.8888888888888888 0.96 Motorbikes 0.9259259259259259 0.9615384615384616 Faces_easy 1.0 0.9 Faces 0.8148148148148148 0.88 watch 0.8148148148148148 0.7096774193548387 Leopards 0.9259259259259259 0.8333333333333334 bonsai 0.5925925925925926 0.45714285714285713 car_side 0.9259259259259259 1.0 ketch 0.8518518518518519 0.5476190476190477 chandelier 0.7777777777777778 0.7 hawksbill 0.8148148148148148 0.36065573770491804 grand_piano 0.7037037037037037 0.9047619047619048 brain 0.8518518518518519 0.6216216216216216 butterfly 0.5925925925925926 0.45714285714285713 helicopter 0.38461538461538464 0.625 menorah 0.7692307692307693 0.7407407407407407 starfish 0.0 nan trilobite 0.9615384615384616 0.8333333333333334 kangaroo 0.5769230769230769 0.4838709677419355 ewer 0.6153846153846154 0.64 buddha 0.6538461538461539 0.4358974358974359 sunflower 0.88 0.7857142857142857 scorpion 0.0 nan revolver 0.72 0.9473684210526315 laptop 0.5 0.8 ibis 0.0 nan
As we expected standardizing images gives the best result. It improved network results by ~7%. What is surprising is that we achieved almost as good results by simply substructing the mean (After repetition of the test, it achieved just a bit lower val accuracy than standarization. As we can see the network performs significantly worse on raw data, so it is really beneficial to perform this simple operation before running images through network. But the difference is not that huge. And as always, each model strugels with certain class, this time it was again scorpion, then brain.
*for convolutional layers
#CNN block params:
filter_numbers = [32]*3
filter_sizes = [(3, 3)]*3
dropout = [best_dropout]*3
pool_sizes = [(2, 2)]*3
#Dense layers params:
dense_sizes = best_dense_sizes
dense_activations = ["sigmoid", "relu"]
activation_functions = [["relu"]*3, ["tanh"]*3,["sigmoid"]*3 ,["elu"]*3]
models = []
names = []
for activations in activation_functions:
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{str(activations[0])}")
compare_models(models=models, names=names)
#The best is relu
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: relu Number of params: 313074 Model name: tanh Number of params: 313074 Model name: sigmoid Number of params: 313074 Model name: elu Number of params: 313074 Traning times: relu: 32.43809366226196 tanh: 20.511286973953247 sigmoid: 42.72777605056763 elu: 32.78076457977295 Best val accuracies: relu: 0.8257686495780945 tanh: 0.6354319453239441 sigmoid: 0.7408491969108582 elu: 0.7101024985313416
relu Label Recall Precision Worst recall for: buddha 0.5384615384615384 Worst precision for: starfish 0.5769230769230769 airplanes 0.8888888888888888 0.8888888888888888 Motorbikes 1.0 1.0 Faces_easy 1.0 1.0 Faces 0.9259259259259259 1.0 watch 0.6666666666666666 0.9 Leopards 1.0 0.9 bonsai 0.7777777777777778 0.8076923076923077 car_side 1.0 0.84375 ketch 0.8148148148148148 0.9166666666666666 chandelier 0.8148148148148148 0.88 hawksbill 0.8518518518518519 0.6052631578947368 grand_piano 0.7777777777777778 1.0 brain 0.7407407407407407 0.6896551724137931 butterfly 0.6666666666666666 0.72 helicopter 0.6538461538461539 0.85 menorah 0.8461538461538461 0.9565217391304348 starfish 0.5769230769230769 0.5769230769230769 trilobite 0.9615384615384616 0.9259259259259259 kangaroo 0.6923076923076923 0.8571428571428571 ewer 0.8846153846153846 0.5897435897435898 buddha 0.5384615384615384 0.7 sunflower 0.92 0.9583333333333334 scorpion 0.8 0.5882352941176471 revolver 0.88 0.9565217391304348 laptop 0.6666666666666666 0.8888888888888888 ibis 0.875 0.65625 tanh Label Recall Precision Worst recall for: butterfly 0.0 Worst precision for: butterfly 0.0 airplanes 0.7777777777777778 0.84 Motorbikes 1.0 0.84375 Faces_easy 0.9259259259259259 0.8333333333333334 Faces 0.7777777777777778 0.9130434782608695 watch 0.7407407407407407 0.625 Leopards 0.9629629629629629 0.7428571428571429 bonsai 0.7777777777777778 0.45652173913043476 car_side 1.0 0.54 ketch 0.48148148148148145 0.6842105263157895 chandelier 0.7037037037037037 0.7037037037037037 hawksbill 0.2962962962962963 0.6666666666666666 grand_piano 0.7777777777777778 0.75 brain 0.5185185185185185 0.7368421052631579 butterfly 0.0 nan helicopter 0.38461538461538464 0.3125 menorah 0.8076923076923077 0.875 starfish 0.2692307692307692 0.5833333333333334 trilobite 0.9615384615384616 0.6097560975609756 kangaroo 0.3076923076923077 0.4444444444444444 ewer 0.46153846153846156 0.3076923076923077 buddha 0.38461538461538464 0.20408163265306123 sunflower 0.96 0.631578947368421 scorpion 0.44 0.3793103448275862 revolver 0.64 0.6956521739130435 laptop 0.0 nan ibis 0.0 nan
/tmp/ipykernel_113926/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
sigmoid Label Recall Precision Worst recall for: scorpion 0.0 Worst precision for: scorpion 0.0 airplanes 0.8888888888888888 0.7741935483870968 Motorbikes 0.9629629629629629 0.9629629629629629 Faces_easy 1.0 0.9310344827586207 Faces 0.8148148148148148 0.9565217391304348 watch 0.8888888888888888 0.8888888888888888 Leopards 0.9259259259259259 0.8064516129032258 bonsai 0.7777777777777778 0.9130434782608695 car_side 1.0 0.84375 ketch 0.8148148148148148 0.88 chandelier 0.7407407407407407 0.7407407407407407 hawksbill 0.6296296296296297 0.6538461538461539 grand_piano 0.7777777777777778 0.7777777777777778 brain 0.7777777777777778 0.75 butterfly 0.5185185185185185 0.7368421052631579 helicopter 0.4230769230769231 0.6111111111111112 menorah 0.8461538461538461 0.9166666666666666 starfish 0.6153846153846154 0.37209302325581395 trilobite 1.0 0.7027027027027027 kangaroo 0.7307692307692307 0.59375 ewer 0.5 0.6190476190476191 buddha 0.38461538461538464 0.7692307692307693 sunflower 0.56 0.9333333333333333 scorpion 0.0 nan revolver 0.68 0.8947368421052632 laptop 0.7083333333333334 0.6296296296296297 ibis 0.875 0.3559322033898305 elu Label Recall Precision Worst recall for: brain 0.0 Worst precision for: brain 0.0 airplanes 0.8148148148148148 0.8461538461538461 Motorbikes 0.9629629629629629 0.896551724137931 Faces_easy 1.0 0.9 Faces 0.8518518518518519 0.8518518518518519 watch 0.7407407407407407 0.9090909090909091 Leopards 0.6296296296296297 0.9444444444444444 bonsai 0.7777777777777778 0.7241379310344828 car_side 1.0 0.9310344827586207 ketch 0.8888888888888888 0.7741935483870968 chandelier 0.9259259259259259 0.3246753246753247 hawksbill 0.6296296296296297 0.5483870967741935 grand_piano 0.8148148148148148 0.9565217391304348 brain 0.0 nan butterfly 0.7037037037037037 0.35185185185185186 helicopter 0.5384615384615384 0.9333333333333333 menorah 0.7692307692307693 0.8 starfish 0.5384615384615384 0.4666666666666667 trilobite 0.7692307692307693 0.9090909090909091 kangaroo 0.6153846153846154 0.42105263157894735 ewer 0.6923076923076923 0.45 buddha 0.46153846153846156 0.48 sunflower 0.52 0.9285714285714286 scorpion 0.2 0.5 revolver 0.76 0.7916666666666666 laptop 0.0 nan ibis 0.4166666666666667 0.7142857142857143
We can clearly ssee that relu function is the best for Conv layers. What is surprising is that its training time was almost the highest. However we can see that it made more epochs and last 15 of them were close to optimum. Which means that it is more stable. Gelu was almost as good as relu, but it was less stable in the end. Sigmoid function also did pretty well. But after talk with Doctor Krzysztof Martyn, we decided to use relu.
best_cnn_activation = "relu"
#Different number of convolutional blocks:
dense_sizes = best_dense_sizes
dense_activations = ["sigmoid", "relu"]
models = []
names = []
for num_of_blocks in range(1, 7):
#CNN block params:
filter_numbers = [32 for x in range(num_of_blocks)]
filter_sizes = [(3, 3) for x in range(num_of_blocks)]
activations = [best_cnn_activation for x in range(num_of_blocks)]
dropout = [best_dropout for x in range(num_of_blocks)]
pool_sizes = [(2, 2) for x in range(num_of_blocks)]
a_model = {"filter_numbers":filter_numbers, "filter_sizes":filter_sizes, "activations":activations,
"dropout":dropout, "pool_sizes": pool_sizes,
"dense_sizes":dense_sizes, "dense_activations":dense_activations}
models.append(a_model)
names.append(f"{num_of_blocks} blocks")
compare_models(models=models, names=names)
--- I'm not iterating data, same data for very model. --- Number of trainable params: Model name: 1 blocks Number of params: 4226482 Model name: 2 blocks Number of params: 1090130 Model name: 3 blocks Number of params: 313074 Model name: 4 blocks Number of params: 125842 Model name: 5 blocks Number of params: 86066 Model name: 6 blocks Number of params: 83154 Traning times: 1 blocks: 21.9439377784729 2 blocks: 24.3140652179718 3 blocks: 29.04506492614746 4 blocks: 28.069286108016968 5 blocks: 29.510061025619507 6 blocks: 27.853984355926514 Best val accuracies: 1 blocks: 0.5212298631668091 2 blocks: 0.6778916716575623 3 blocks: 0.7496339678764343 4 blocks: 0.7467057108879089 5 blocks: 0.6002928018569946 6 blocks: 0.4919472932815552
/tmp/ipykernel_113926/3680036453.py:52: RuntimeWarning: invalid value encountered in true_divide precision = np.diag(cm) / np.sum(cm, axis = 0)
1 blocks Label Recall Precision Worst recall for: brain 0.0 Worst precision for: brain 0.0 airplanes 0.8888888888888888 0.8 Motorbikes 0.8888888888888888 1.0 Faces_easy 0.9629629629629629 0.7878787878787878 Faces 0.5925925925925926 0.6153846153846154 watch 0.6666666666666666 0.5454545454545454 Leopards 0.9259259259259259 0.8620689655172413 bonsai 0.5185185185185185 0.4666666666666667 car_side 0.8518518518518519 0.8518518518518519 ketch 0.7407407407407407 0.6451612903225806 chandelier 0.5925925925925926 0.38095238095238093 hawksbill 0.5185185185185185 0.4 grand_piano 0.5555555555555556 0.8333333333333334 brain 0.0 nan butterfly 0.37037037037037035 0.45454545454545453 helicopter 0.5769230769230769 0.3488372093023256 menorah 0.5 0.6190476190476191 starfish 0.6153846153846154 0.26229508196721313 trilobite 0.6538461538461539 0.7083333333333334 kangaroo 0.7692307692307693 0.20618556701030927 ewer 0.0 nan buddha 0.0 nan sunflower 0.0 nan scorpion 0.0 nan revolver 0.72 0.8181818181818182 laptop 0.5 0.34285714285714286 ibis 0.0 nan 2 blocks Label Recall Precision Worst recall for: Faces_easy 0.0 Worst precision for: Faces_easy 0.0 airplanes 0.9629629629629629 0.8125 Motorbikes 0.9259259259259259 1.0 Faces_easy 0.0 nan Faces 0.9259259259259259 0.5681818181818182 watch 0.6296296296296297 0.8095238095238095 Leopards 1.0 0.8709677419354839 bonsai 0.6296296296296297 0.7391304347826086 car_side 0.9629629629629629 0.9629629629629629 ketch 0.7407407407407407 0.8333333333333334 chandelier 0.6296296296296297 0.7083333333333334 hawksbill 0.48148148148148145 0.5909090909090909 grand_piano 0.7037037037037037 0.8260869565217391 brain 0.6666666666666666 0.6 butterfly 0.48148148148148145 0.5909090909090909 helicopter 0.5384615384615384 0.56 menorah 0.6538461538461539 0.8095238095238095 starfish 0.5384615384615384 0.3333333333333333 trilobite 0.9230769230769231 0.96 kangaroo 0.5384615384615384 0.6363636363636364 ewer 0.7692307692307693 0.32786885245901637 buddha 0.7692307692307693 0.7142857142857143 sunflower 0.72 0.8571428571428571 scorpion 0.56 0.4827586206896552 revolver 0.84 0.875 laptop 0.4583333333333333 0.7857142857142857 ibis 0.5 0.5217391304347826 3 blocks Label Recall Precision Worst recall for: butterfly 0.0 Worst precision for: butterfly 0.0 airplanes 0.9259259259259259 0.9259259259259259 Motorbikes 0.9629629629629629 1.0 Faces_easy 0.9259259259259259 0.8064516129032258 Faces 0.8148148148148148 0.8148148148148148 watch 0.7777777777777778 0.6774193548387096 Leopards 0.5925925925925926 0.8421052631578947 bonsai 0.6296296296296297 0.6538461538461539 car_side 0.8888888888888888 0.9230769230769231 ketch 0.8148148148148148 0.7586206896551724 chandelier 0.5925925925925926 0.8 hawksbill 0.5555555555555556 0.5357142857142857 grand_piano 0.6666666666666666 0.75 brain 0.7037037037037037 0.7916666666666666 butterfly 0.0 nan helicopter 0.46153846153846156 0.7058823529411765 menorah 0.6538461538461539 0.8947368421052632 starfish 0.6538461538461539 0.6071428571428571 trilobite 0.9615384615384616 0.8928571428571429 kangaroo 0.7307692307692307 0.3333333333333333 ewer 0.6538461538461539 0.5151515151515151 buddha 0.7307692307692307 0.3584905660377358 sunflower 0.76 0.8636363636363636 scorpion 0.4 0.35714285714285715 revolver 0.88 0.7857142857142857 laptop 0.4583333333333333 1.0 ibis 0.6666666666666666 0.7619047619047619 4 blocks Label Recall Precision Worst recall for: scorpion 0.0 Worst precision for: scorpion 0.0 airplanes 0.7777777777777778 0.875 Motorbikes 0.9629629629629629 0.9629629629629629 Faces_easy 0.9629629629629629 0.8125 Faces 0.7777777777777778 0.84 watch 0.7777777777777778 0.5675675675675675 Leopards 0.8518518518518519 0.92 bonsai 0.8148148148148148 0.7333333333333333 car_side 0.9629629629629629 0.7878787878787878 ketch 0.7407407407407407 0.8333333333333334 chandelier 0.6666666666666666 0.6923076923076923 hawksbill 0.7037037037037037 0.7037037037037037 grand_piano 0.7037037037037037 1.0 brain 0.7037037037037037 0.6551724137931034 butterfly 0.6296296296296297 0.7391304347826086 helicopter 0.6538461538461539 0.5483870967741935 menorah 0.6538461538461539 0.8095238095238095 starfish 0.6538461538461539 0.5666666666666667 trilobite 1.0 0.7647058823529411 kangaroo 0.8846153846153846 0.6764705882352942 ewer 0.7307692307692307 0.5 buddha 0.6538461538461539 0.9444444444444444 sunflower 0.76 0.9047619047619048 scorpion 0.0 nan revolver 0.96 0.7741935483870968 laptop 0.625 0.75 ibis 0.625 0.625 5 blocks Label Recall Precision Worst recall for: chandelier 0.0 Worst precision for: chandelier 0.0 airplanes 0.8148148148148148 0.8461538461538461 Motorbikes 1.0 0.7941176470588235 Faces_easy 0.9629629629629629 0.8666666666666667 Faces 0.8888888888888888 0.7741935483870968 watch 0.4074074074074074 0.275 Leopards 0.9259259259259259 0.7575757575757576 bonsai 0.7777777777777778 0.5 car_side 0.9629629629629629 0.8666666666666667 ketch 0.7777777777777778 0.5 chandelier 0.0 nan hawksbill 0.5555555555555556 0.5769230769230769 grand_piano 0.7407407407407407 0.9523809523809523 brain 0.8518518518518519 0.5348837209302325 butterfly 0.5185185185185185 0.6086956521739131 helicopter 0.5769230769230769 0.5357142857142857 menorah 0.6923076923076923 0.6 starfish 0.6153846153846154 0.24242424242424243 trilobite 0.7307692307692307 1.0 kangaroo 0.5769230769230769 0.375 ewer 0.0 nan buddha 0.0 nan sunflower 0.36 1.0 scorpion 0.0 nan revolver 0.96 0.5454545454545454 laptop 0.6666666666666666 0.6153846153846154 ibis 0.0 nan 6 blocks Label Recall Precision Worst recall for: hawksbill 0.0 Worst precision for: hawksbill 0.0 airplanes 0.9629629629629629 0.3561643835616438 Motorbikes 0.5925925925925926 1.0 Faces_easy 0.9259259259259259 0.8333333333333334 Faces 0.8148148148148148 0.9166666666666666 watch 0.7407407407407407 0.29850746268656714 Leopards 0.9629629629629629 0.48148148148148145 bonsai 0.8518518518518519 0.4423076923076923 car_side 0.9629629629629629 0.8125 ketch 0.8518518518518519 0.7666666666666667 chandelier 0.6296296296296297 0.11564625850340136 hawksbill 0.0 nan grand_piano 0.5925925925925926 1.0 brain 0.0 nan butterfly 0.25925925925925924 1.0 helicopter 0.3076923076923077 0.4444444444444444 menorah 0.0 nan starfish 0.0 nan trilobite 0.9230769230769231 0.4528301886792453 kangaroo 0.0 nan ewer 0.0 nan buddha 0.0 nan sunflower 0.52 0.65 scorpion 0.0 nan revolver 0.76 0.7916666666666666 laptop 0.4166666666666667 0.5 ibis 0.0 nan
It seems that initial requirement of the project was correct. After multitude of test - 3 blocks always proved to be the best. However 4 blocks are just slightly worse and have smoother learning curve. We could not reach as high score as in activation function testing. Probably because in previous one, we were lucky to have a starting model that allowed us to reach so high accuracy. We deduct that data and first initialization overwall plays the biggest role in traning this kind of neural network.
After the tests above, we think we can achieve an accuracy of around 80% which seems good, not great. Still there is a room for improvment and tests of other hyper parameters. Anyhow, the experiment taught as lot, even though Kacper already had some experience with neural networks.